diff --git a/pom.xml b/pom.xml index a7806470a1..b4acbb30f7 100644 --- a/pom.xml +++ b/pom.xml @@ -69,15 +69,12 @@ 2.7.10 1.7.5 4.4.1 - 1.2.1 2.7.1 - 0.94.25 - 0.96.2 - ${hbase096.core.version}-hadoop2 0.98.2 ${hbase098.core.version}-hadoop2 1.0.2 ${hbase100.core.version} + 1.1.1 1.9.2 2.3.0 - + titan-dist-hadoop-2 + ${project.groupId} diff --git a/titan-dist/titan-dist-hadoop-1/pom.xml b/titan-dist/titan-dist-hadoop-1/pom.xml deleted file mode 100644 index b91030b798..0000000000 --- a/titan-dist/titan-dist-hadoop-1/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-dist - 1.0.1-SNAPSHOT - ../pom.xml - - pom - titan-dist-hadoop-1 - Titan-Dist: Archive with Hadoop 1 - http://thinkaurelius.github.com/titan/ - - - hadoop1 - ${project.parent.basedir}/src/assembly/descriptor - ${project.parent.basedir}/src/assembly/static - ${project.parent.basedir}/src/assembly/resources - ${project.parent.build.directory}/cfilter - ${project.parent.parent.basedir} - - - - - ${project.groupId} - titan-all - ${project.version} - pom - - - org.apache.hadoop - hadoop-client - - - org.apache.hbase - hbase-client - - - - ${project.groupId} - titan-solr - - - - - org.apache.hbase - hbase-client - ${hbase098.core.version}-hadoop1 - - - - org.jruby - jruby-complete - - - - - org.apache.hadoop - hadoop-client - ${hadoop1.version} - - - - - - - maven-assembly-plugin - - - - org.codehaus.mojo - exec-maven-plugin - - - generate-titan-gremlin-imports - generate-resources - - - - - - org.codehaus.mojo - properties-maven-plugin - - - - maven-gpg-plugin - - - - - - - aurelius-release - - - - maven-failsafe-plugin - - - - org.codehaus.mojo - wagon-maven-plugin - - - - maven-resources-plugin - - - filter-cassandra-murmur-config - process-test-resources - - - filter-cassandra-bop-config - process-test-resources - - - filter-expect-scripts - process-test-resources - - - filter-test-resources - process-test-resources - - - copy-test-cfiles - process-test-resources - - - - - - - - - - - - dev-install-hadoop1 - - - - dev.hadoop - 1 - - - - - - - maven-clean-plugin - - - clean-dev-dirs-hadoop - clean - - - - - - maven-assembly-plugin - - - install-dev-dirs-hadoop - install - - - - - - - - - dev-install-hadoop1-by-default - - - - - !dev.hadoop - - - - - - - maven-clean-plugin - - - clean-dev-dirs-hadoop - clean - - - - - - maven-assembly-plugin - - - install-dev-dirs-hadoop - install - - - - - - - - diff --git a/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml b/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml deleted file mode 100644 index 399f49ea64..0000000000 --- a/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - titan-${project.version}-${hadoop.version.tag} - titan-${project.version}-${hadoop.version.tag} - - - ${assembly.descriptor.dir}/archive.xml - - - - - src/assembly/static - / - false - - - diff --git a/titan-dist/titan-dist-hadoop-2/pom.xml b/titan-dist/titan-dist-hadoop-2/pom.xml index fa5fdaa041..88841f8398 100644 --- a/titan-dist/titan-dist-hadoop-2/pom.xml +++ b/titan-dist/titan-dist-hadoop-2/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.9.0-SNAPSHOT + 1.0.1-SNAPSHOT ../pom.xml pom @@ -24,13 +24,19 @@ org.apache.hbase hbase-client - ${hbase098.version} + ${hbase111.version} org.apache.hbase hbase-server + ${hbase111.version} - ${hbase098.version} + + + com.lmax + disruptor + + @@ -151,7 +157,7 @@ -classpath com.thinkaurelius.titan.example.GraphOfTheGodsFactory - ${project.build.directory}/conf/titan-berkeleydb-es.properties + ${project.build.directory}/conf/titan-berkeleyje-es.properties diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index 363c4f0c50..3456d7bb2e 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -20,7 +20,6 @@ titan-hadoop-core - titan-hadoop-1 titan-hadoop-2 titan-hadoop diff --git a/titan-hadoop-parent/titan-hadoop-1/pom.xml b/titan-hadoop-parent/titan-hadoop-1/pom.xml deleted file mode 100644 index 69cb61d6b9..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/pom.xml +++ /dev/null @@ -1,112 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-hadoop-parent - 1.0.1-SNAPSHOT - ../pom.xml - - titan-hadoop-1 - Titan-Hadoop: 1.x Compatibility Shim - http://thinkaurelius.github.com/titan/ - - - ${basedir}/../.. - - - - - org.apache.hadoop - hadoop-core - ${hadoop1.version} - - - javax.servlet - jsp-api - - - javax.servlet - servlet-api - - - org.mortbay.jetty - servlet-api-2.5 - - - org.mortbay.jetty - servlet-api - - - - - ${project.groupId} - titan-hadoop-core - ${project.version} - - - ${project.groupId} - titan-hadoop-core - ${project.version} - tests - test - - - ${project.groupId} - titan-hadoop-core - ${project.version} - shared-resources - test - true - - - org.apache.hbase - hbase-client - ${hbase098.core.version}-hadoop1 - true - test - - - - org.jruby - jruby-complete - - - - - org.apache.mrunit - mrunit - ${mrunit.version} - hadoop1 - - - - ${project.groupId} - titan-es - ${project.version} - tests - test - - - - - - - maven-assembly-plugin - - - maven-dependency-plugin - - - maven-failsafe-plugin - - - maven-surefire-plugin - - - - diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java deleted file mode 100644 index 4432044208..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.hadoop.config.job.AbstractDistCacheConfigurer; -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Job; - -import java.io.IOException; - -public class DistCacheConfigurer extends AbstractDistCacheConfigurer implements JobClasspathConfigurer { - - public DistCacheConfigurer(String mapredJarFilename) { - super(mapredJarFilename); - } - - @Override - public void configure(Job job) throws IOException { - - for (Path p : getLocalPaths()) { - Configuration conf = job.getConfiguration(); - FileSystem jobFS = FileSystem.get(conf); - FileSystem localFS = FileSystem.getLocal(conf); - Path stagedPath = uploadFileIfNecessary(localFS, p, jobFS); - DistributedCache.addFileToClassPath(stagedPath, conf, jobFS); - } - - // We don't really need to set a mapred job jar here, - // but doing so suppresses a warning - String mj = getMapredJar(); - if (null != mj) - job.getConfiguration().set(Hadoop1Compat.CFG_JOB_JAR, mj); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java deleted file mode 100644 index b47025a882..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.core.TitanException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics; -import com.thinkaurelius.titan.graphdb.configuration.TitanConstants; -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import com.thinkaurelius.titan.hadoop.formats.cassandra.CassandraBinaryInputFormat; -import com.thinkaurelius.titan.hadoop.scan.HadoopVertexScanMapper; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.*; -import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; - -import com.thinkaurelius.titan.hadoop.compat.HadoopCompat; - -import java.io.IOException; - -public class Hadoop1Compat implements HadoopCompat { - - static final String CFG_SPECULATIVE_MAPS = "mapred.map.tasks.speculative.execution"; - static final String CFG_SPECULATIVE_REDUCES = "mapred.reduce.tasks.speculative.execution"; - static final String CFG_JOB_JAR = "mapred.jar"; - - @Override - public TaskAttemptContext newTask(Configuration c, TaskAttemptID t) { - return new TaskAttemptContext(c, t); - } - - @Override - public String getSpeculativeMapConfigKey() { - return CFG_SPECULATIVE_MAPS; - } - - @Override - public String getSpeculativeReduceConfigKey() { - return CFG_SPECULATIVE_REDUCES; - } - - @Override - public String getMapredJarConfigKey() { - return CFG_JOB_JAR; - } - - @Override - public long getContextCounter(TaskInputOutputContext context, String group, String name) { - return context.getCounter(group, name).getValue(); - } - - @Override - public void incrementContextCounter(TaskInputOutputContext context, - String group, String name, long incr) { - context.getCounter(group, name).increment(incr); - } - - @Override - public Configuration getContextConfiguration(TaskAttemptContext context) { - return context.getConfiguration(); - } - - @Override - public JobClasspathConfigurer newMapredJarConfigurer(String mapredJarPath) { - return new MapredJarConfigurer(mapredJarPath); - } - - @Override - public JobClasspathConfigurer newDistCacheConfigurer() { - return new DistCacheConfigurer("titan-hadoop-core-" + TitanConstants.VERSION + ".jar"); - } - - @Override - public Configuration getJobContextConfiguration(JobContext context) { - return context.getConfiguration(); - } - - @Override - public Configuration newImmutableConfiguration(Configuration base) { - return new ImmutableConfiguration(base); - } - - @Override - public ScanMetrics getMetrics(Counters c) { - return new Hadoop1CountersScanMetrics(c); - } - - @Override - public String getJobFailureString(Job j) { - return j.toString(); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java deleted file mode 100644 index a9c236d3fc..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics; -import com.thinkaurelius.titan.hadoop.scan.HadoopContextScanMetrics; -import org.apache.hadoop.mapreduce.Counters; - -public class Hadoop1CountersScanMetrics implements ScanMetrics { - - private final Counters counters; - - public Hadoop1CountersScanMetrics(Counters counters) { - this.counters = counters; - } - - @Override - public long getCustom(String metric) { - return counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).getValue(); - } - - @Override - public void incrementCustom(String metric, long delta) { - counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).increment(delta); - } - - @Override - public void incrementCustom(String metric) { - incrementCustom(metric, 1L); - } - - @Override - public long get(Metric metric) { - return counters.getGroup(HadoopContextScanMetrics.STANDARD_COUNTER_GROUP).findCounter(metric.name()).getValue(); - } - - @Override - public void increment(Metric metric) { - counters.getGroup(HadoopContextScanMetrics.STANDARD_COUNTER_GROUP).findCounter(metric.name()).increment(1L); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java deleted file mode 100644 index b85647f018..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java +++ /dev/null @@ -1,283 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; - -import java.io.*; -import java.net.URL; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -public class ImmutableConfiguration extends Configuration { - - private final Configuration encapsulated; - - public ImmutableConfiguration(final Configuration encapsulated) { - this.encapsulated = encapsulated; - } - - public static void addDefaultResource(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(URL url) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(Path file) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(InputStream in) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void reloadConfiguration() { - //throw new UnsupportedOperationException("This configuration instance is immutable"); - encapsulated.reloadConfiguration(); // allowed to simplify testing - } - - @Override - public String get(String name) { - return encapsulated.get(name); - } - - @Override - public String getRaw(String name) { - return encapsulated.getRaw(name); - } - - @Override - public void set(String name, String value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void unset(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void setIfUnset(String name, String value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public String get(String name, String defaultValue) { - return encapsulated.get(name, defaultValue); - } - - @Override - public int getInt(String name, int defaultValue) { - return encapsulated.getInt(name, defaultValue); - } - - @Override - public void setInt(String name, int value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public long getLong(String name, long defaultValue) { - return encapsulated.getLong(name, defaultValue); - } - - @Override - public void setLong(String name, long value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public float getFloat(String name, float defaultValue) { - return encapsulated.getFloat(name, defaultValue); - } - - @Override - public void setFloat(String name, float value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public boolean getBoolean(String name, boolean defaultValue) { - return encapsulated.getBoolean(name, defaultValue); - } - - @Override - public void setBoolean(String name, boolean value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void setBooleanIfUnset(String name, boolean value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public > void setEnum(String name, T value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public > T getEnum(String name, T defaultValue) { - return encapsulated.getEnum(name, defaultValue); - } - - @Override - public IntegerRanges getRange(String name, String defaultValue) { - return encapsulated.getRange(name, defaultValue); - } - - @Override - public Collection getStringCollection(String name) { - return encapsulated.getStringCollection(name); - } - - @Override - public String[] getStrings(String name) { - return encapsulated.getStrings(name); - } - - @Override - public String[] getStrings(String name, String... defaultValue) { - return encapsulated.getStrings(name, defaultValue); - } - - @Override - public void setStrings(String name, String... values) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Class getClassByName(String name) throws ClassNotFoundException { - return encapsulated.getClassByName(name); - } - - @Override - public Class[] getClasses(String name, Class... defaultValue) { - return encapsulated.getClasses(name, defaultValue); - } - - @Override - public Class getClass(String name, Class defaultValue) { - return encapsulated.getClass(name, defaultValue); - } - - @Override - public Class getClass(String name, Class defaultValue, Class xface) { - return encapsulated.getClass(name, defaultValue, xface); - } - - @Override - public List getInstances(String name, Class xface) { - return encapsulated.getInstances(name, xface); - } - - @Override - public void setClass(String name, Class theClass, Class xface) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Path getLocalPath(String dirsProp, String path) throws IOException { - return encapsulated.getLocalPath(dirsProp, path); - } - - @Override - public File getFile(String dirsProp, String path) throws IOException { - return encapsulated.getFile(dirsProp, path); - } - - @Override - public URL getResource(String name) { - return encapsulated.getResource(name); - } - - @Override - public InputStream getConfResourceAsInputStream(String name) { - return encapsulated.getConfResourceAsInputStream(name); - } - - @Override - public Reader getConfResourceAsReader(String name) { - return encapsulated.getConfResourceAsReader(name); - } - - @Override - public int size() { - return encapsulated.size(); - } - - @Override - public void clear() { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Iterator> iterator() { - return encapsulated.iterator(); - } - - @Override - public void writeXml(OutputStream out) throws IOException { - encapsulated.writeXml(out); - } - - @Override - public void writeXml(Writer out) throws IOException { - encapsulated.writeXml(out); - } - - public static void dumpConfiguration(Configuration config, Writer out) throws IOException { - Configuration.dumpConfiguration(config, out); - } - - @Override - public ClassLoader getClassLoader() { - return encapsulated.getClassLoader(); - } - - @Override - public void setClassLoader(ClassLoader classLoader) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public String toString() { - return encapsulated.toString(); - } - - @Override - public void setQuietMode(boolean quietmode) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - public static void main(String[] args) throws Exception { - Configuration.main(args); - } - - @Override - public void readFields(DataInput in) throws IOException { - encapsulated.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - encapsulated.write(out); - } - - @Override - public Map getValByRegex(String regex) { - return encapsulated.getValByRegex(regex); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java deleted file mode 100644 index 2be1c5f013..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import org.apache.hadoop.mapreduce.Job; - -import java.io.IOException; - -public class MapredJarConfigurer implements JobClasspathConfigurer { - - private final String mapredJar; - - public MapredJarConfigurer(String mapredJar) { - this.mapredJar = mapredJar; - } - - @Override - public void configure(Job job) throws IOException { - job.getConfiguration().set(Hadoop1Compat.CFG_JOB_JAR, mapredJar); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java deleted file mode 100644 index 2134ed222d..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; - -public class TitanH1OutputCommitter extends OutputCommitter { - private final TitanH1OutputFormat tof; - - public TitanH1OutputCommitter(TitanH1OutputFormat tof) { - this.tof = tof; - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - - } - - @Override - public void setupTask(TaskAttemptContext taskContext) throws IOException { - - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { - return tof.hasModifications(taskContext.getTaskAttemptID()); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) throws IOException { - tof.commit(taskContext.getTaskAttemptID()); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) throws IOException { - tof.abort(taskContext.getTaskAttemptID()); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java deleted file mode 100644 index a898ed5600..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.thinkaurelius.titan.core.TitanFactory; -import com.thinkaurelius.titan.graphdb.database.StandardTitanGraph; -import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx; -import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; -import com.thinkaurelius.titan.hadoop.config.TitanHadoopConfiguration; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; -import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil; -import org.apache.tinkerpop.gremlin.process.computer.VertexProgram; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -public class TitanH1OutputFormat extends OutputFormat { - - private static final Logger log = LoggerFactory.getLogger(TitanH1OutputFormat.class); - - private final ConcurrentMap transactions = new ConcurrentHashMap<>(); - - private StandardTitanGraph graph; - - private Set persistableKeys; - - @Override - public RecordWriter getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - - synchronized (this) { - if (null == graph) { - Configuration hadoopConf = taskAttemptContext.getConfiguration(); - ModifiableHadoopConfiguration mhc = - ModifiableHadoopConfiguration.of(TitanHadoopConfiguration.MAPRED_NS, hadoopConf); - graph = (StandardTitanGraph) TitanFactory.open(mhc.getTitanGraphConf()); - } - } - - // Special case for a TP3 vertex program: persist only those properties whose keys are - // returned by VertexProgram.getComputeKeys() - if (null == persistableKeys) { - try { - persistableKeys = VertexProgram.createVertexProgram(graph, - ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration())).getElementComputeKeys(); - log.debug("Set persistableKeys={}", Joiner.on(",").join(persistableKeys)); - } catch (Exception e) { - log.debug("Unable to detect or instantiate vertex program", e); - persistableKeys = ImmutableSet.of(); - } - } - - StandardTitanTx tx = transactions.computeIfAbsent(taskAttemptContext.getTaskAttemptID(), - id -> (StandardTitanTx)graph.newTransaction()); - return new TitanH1RecordWriter(taskAttemptContext, tx, persistableKeys); - } - - @Override - public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException { - // TODO check output configuration for minimum set of keys here? - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws IOException, - InterruptedException { - return new TitanH1OutputCommitter(this); - } - - void commit(TaskAttemptID id) { - StandardTitanTx tx = transactions.remove(id); - if (null == tx) { - log.warn("Detected concurrency in task commit"); - return; - } - tx.commit(); - } - - void abort(TaskAttemptID id) { - StandardTitanTx tx = transactions.remove(id); - if (null == tx) { - log.warn("Detected concurrency in task abort"); - return; - } - tx.rollback(); - } - - boolean hasModifications(TaskAttemptID id) { - StandardTitanTx tx = transactions.get(id); - // if tx is null, something is horribly wrong - return tx.hasModifications(); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java deleted file mode 100644 index b2f4f28e8b..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Set; - -public class TitanH1RecordWriter extends RecordWriter { - - private static final Logger log = LoggerFactory.getLogger(TitanH1RecordWriter.class); - - private final TaskAttemptContext taskAttemptContext; - private final StandardTitanTx tx; - private final Set persistableKeys; - - public TitanH1RecordWriter(TaskAttemptContext taskAttemptContext, StandardTitanTx tx, Set persistableKeys) { - this.taskAttemptContext = taskAttemptContext; - this.tx = tx; - this.persistableKeys = persistableKeys; - } - - @Override - public void write(NullWritable key, VertexWritable value) throws IOException, InterruptedException { - // TODO tolerate possibility that concurrent OLTP activity has deleted the vertex? maybe configurable... - Object vertexID = value.get().id(); - Vertex vertex = tx.vertices(vertexID).next(); - Iterator> vpIter = value.get().properties(); - while (vpIter.hasNext()) { - VertexProperty vp = vpIter.next(); - if (!persistableKeys.isEmpty() && !persistableKeys.contains(vp.key())) { - log.debug("[vid {}] skipping key {}", vertexID, vp.key()); - continue; - } - vertex.property(vp.key(), vp.value()); - log.debug("[vid {}] set {}={}", vertexID, vp.key(), vp.value()); - } - } - - @Override - public void close(TaskAttemptContext context) throws IOException, InterruptedException { - // commit/rollback happens in TitanOutputCommitter - // nothing to do here - } -} diff --git a/titan-hadoop-parent/titan-hadoop/pom.xml b/titan-hadoop-parent/titan-hadoop/pom.xml index c0a5e831f0..38138d2dfe 100644 --- a/titan-hadoop-parent/titan-hadoop/pom.xml +++ b/titan-hadoop-parent/titan-hadoop/pom.xml @@ -22,12 +22,6 @@ ${project.version} true - - ${project.groupId} - titan-hadoop-1 - ${project.version} - true - ${project.groupId} titan-hadoop-2 diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 0b0010d6da..9c0b041917 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -13,10 +13,9 @@ titan-hbase-core - titan-hbase-094 - titan-hbase-096 titan-hbase-098 titan-hbase-10 + titan-hbase-11 titan-hbase diff --git a/titan-hbase-parent/titan-hbase-094/.gitignore b/titan-hbase-parent/titan-hbase-094/.gitignore deleted file mode 100644 index 89540d8235..0000000000 --- a/titan-hbase-parent/titan-hbase-094/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/dependency-reduced-pom.xml diff --git a/titan-hbase-parent/titan-hbase-094/pom.xml b/titan-hbase-parent/titan-hbase-094/pom.xml deleted file mode 100644 index f176e66a86..0000000000 --- a/titan-hbase-parent/titan-hbase-094/pom.xml +++ /dev/null @@ -1,352 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-hbase-parent - 1.0.1-SNAPSHOT - ../pom.xml - - titan-hbase-094 - Titan-HBase: 0.94 Compatibility Shim - http://thinkaurelius.github.com/titan/ - - - ${hadoop1.version} - 1.6.5 - 1.0 - ${project.parent.parent.basedir} - - - - - ${project.groupId} - titan-hbase-core - ${project.version} - - - commons-net - commons-net - - - - - ${project.groupId} - titan-hbase-core - ${project.version} - tests - test - - - org.apache.hbase - hbase - ${hbase094.version} - true - - - avro - org.apache.avro - - - jruby-complete - org.jruby - - - asm - asm - - - - - com.google.protobuf - protobuf-java - true - - - - - - - - - com.google.protobuf - protobuf-java - 2.4.0a - - - - - - src/main/java - - - ${project.basedir}/../common/test/resources - true - - - - - maven-resources-plugin - - - filter-hbase-conf - - - - - maven-dependency-plugin - - - generate-cached-classpath - - - unpack-common-test-classes - - - - - maven-assembly-plugin - - - copy-hbase-bin - - - - - maven-clean-plugin - - - clean-hbase-files - - - clean-shade-pom - clean - - clean - - - true - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - - maven-deploy-plugin - - - default-deploy - none - - - - - maven-shade-plugin - 2.2 - - - - shade-hbase-094-libs - package - - shade - - - true - false - false - - false - - - com.google.protobuf:protobuf-java - - - - - - com.google.protobuf - com.google.protobuf.protobuf240a - - - - - - - - - - - - - - hadoop1 - - - - !hadoop.profile - - - - - - org.apache.hadoop - hadoop-core - ${hadoop1.version} - true - - - jruby-complete - org.jruby - ${jruby.version} - test - - - - - - hadoop2 - - - - hadoop.profile - 2.0 - - - - - ${hadoop2.version} - - - - - org.apache.hadoop - hadoop-annotations - ${hadoop2.version} - - - org.apache.hadoop - hadoop-hdfs - ${hadoop2.version} - - - org.apache.hadoop - hadoop-client - ${hadoop2.version} - - - - - - mapr - - - - hadoop.profile - mapr-1.0 - - - - - 0.94.12-mapr-1310 - 1.0.3-mapr-3.0.2 - 3.4.5 - -Dzookeeper.version=${zookeeper.version} - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - com.mapr.fs - mapr-hbase - ${hadoop.version} - - - - - - org.apache.hbase - hbase - ${hbase094.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - commons-configuration - commons-configuration - 1.8 - - - org.mortbay.jetty - jetty-util - 6.1.26 - - - - - - diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java deleted file mode 100644 index 5684a5abe0..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2004 The Guava Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.common.io; - -import com.google.common.annotations.Beta; - -import java.io.OutputStream; - -/** - * Implementation of {@link OutputStream} that simply discards written bytes. - * - * @author Spencer Kimball - * @since 1.0 - */ -@Beta -public final class NullOutputStream extends OutputStream { - /** Discards the specified byte. */ - @Override public void write(int b) { - } - - /** Discards the specified byte array. */ - @Override public void write(byte[] b, int off, int len) { - } -} - diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java deleted file mode 100644 index fd3726806e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; - -public class HBaseAdmin0_94 implements AdminMask -{ - - private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_94.class); - - private final HBaseAdmin adm; - - public HBaseAdmin0_94(HBaseAdmin adm) - { - this.adm = adm; - } - - @Override - public void clearTable(String tableName, long timestamp) throws IOException - { - if (!adm.tableExists(tableName)) { - log.debug("clearStorage() called before table {} was created, skipping.", tableName); - return; - } - -// long before = System.currentTimeMillis(); -// try { -// adm.disableTable(tableName); -// adm.deleteTable(tableName); -// } catch (IOException e) { -// throw new PermanentBackendException(e); -// } -// ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); -// long after = System.currentTimeMillis(); -// logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before); - - - // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than - // disabling and deleting tables. - HTable table = null; - - try { - table = new HTable(adm.getConfiguration(), tableName); - - Scan scan = new Scan(); - scan.setBatch(100); - scan.setCacheBlocks(false); - scan.setCaching(2000); - scan.setTimeRange(0, Long.MAX_VALUE); - scan.setMaxVersions(1); - - ResultScanner scanner = null; - - try { - scanner = table.getScanner(scan); - - for (Result res : scanner) { - Delete d = new Delete(res.getRow()); - - d.setTimestamp(timestamp); - table.delete(d); - } - } finally { - IOUtils.closeQuietly(scanner); - } - } finally { - IOUtils.closeQuietly(table); - } - } - - @Override - public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException - { - return adm.getTableDescriptor(tableName.getBytes()); - } - - @Override - public boolean tableExists(String tableName) throws IOException - { - return adm.tableExists(tableName); - } - - @Override - public void createTable(HTableDescriptor desc) throws IOException - { - adm.createTable(desc); - } - - @Override - public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException - { - adm.createTable(desc, startKey, endKey, numRegions); - } - - @Override - public int getEstimatedRegionServerCount() - { - int serverCount = -1; - try { - serverCount = adm.getClusterStatus().getServers().size(); - log.debug("Read {} servers from HBase ClusterStatus", serverCount); - } catch (IOException e) { - log.debug("Unable to retrieve HBase cluster status", e); - } - return serverCount; - } - - @Override - public void disableTable(String tableName) throws IOException - { - adm.disableTable(tableName); - } - - @Override - public void enableTable(String tableName) throws IOException - { - adm.enableTable(tableName); - } - - @Override - public boolean isTableDisabled(String tableName) throws IOException - { - return adm.isTableDisabled(tableName); - } - - @Override - public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException - { - adm.addColumn(tableName, columnDescriptor); - } - - @Override - public void close() throws IOException - { - adm.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java deleted file mode 100644 index 26238c1d16..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.io.hfile.Compression; - -public class HBaseCompat0_94 implements HBaseCompat { - - @Override - public void setCompression(HColumnDescriptor cd, String algo) { - cd.setCompressionType(Compression.Algorithm.valueOf(algo)); - } - - @Override - public HTableDescriptor newTableDescriptor(String tableName) { - return new HTableDescriptor(tableName); - } - - @Override - public ConnectionMask createConnection(Configuration conf) throws IOException - { - return new HConnection0_94(HConnectionManager.createConnection(conf)); - } - - @Override - public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc) - { - tdesc.addFamily(cdesc); - } - - @Override - public void setTimestamp(Delete d, long timestamp) - { - d.setTimestamp(timestamp); - } - -} \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb deleted file mode 100755 index 8887a4574c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env hbase-jruby -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Prints the hostname of the machine running the active master. - -include Java -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.ServerName -import org.apache.hadoop.hbase.zookeeper.ZKUtil -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher - -# disable debug/info logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) -org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) - -config = HBaseConfiguration.create - -zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) -begin - master_address = ZKUtil.getData(zk, zk.masterAddressZNode) - if master_address - puts ServerName.parseVersionedServerName(master_address).getHostname() - else - puts 'Master not running' - end -ensure - zk.close() -end - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh deleted file mode 100755 index 9a62440064..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2011 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Move regions off a server then stop it. Optionally restart and reload. -# Turn off the balancer before running this script. -function usage { - echo "Usage: graceful_stop.sh [--config ] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " debug Print helpful debug information" - echo " hostname Hostname of server we are to stop" - exit 1 -} - -if [ $# -lt 1 ]; then - usage -fi - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh -# Get arguments -restart= -reload= -debug= -thrift= -rest= -while [ $# -gt 0 ] -do - case "$1" in - --thrift) thrift=true; shift;; - --rest) rest=true; shift;; - --restart) restart=true; shift;; - --reload) reload=true; shift;; - --debug) debug="--debug"; shift;; - --) shift; break;; - -*) usage ;; - *) break;; # terminate while loop - esac -done - -# "$@" contains the rest. Must be at least the hostname left. -if [ $# -lt 1 ]; then - usage -fi - -hostname=$1 -filename="/tmp/$hostname" -# Run the region mover script. -echo "Disabling balancer! (if required)" -HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` -echo "Previous balancer state was $HBASE_BALANCER_STATE" -echo "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname -echo "Unloaded $hostname region(s)" -# Stop the server. Have to put hostname into its own little file for hbase-daemons.sh -hosts="/tmp/$(basename $0).$$.tmp" -echo $hostname >> $hosts -if [ "$thrift" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift -fi -if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest -fi -"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver -if [ "$restart" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver - if [ "$thrift" != "" ]; then - # -b 0.0.0.0 says listen on all interfaces rather than just default. - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 - fi - if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest - fi - if [ "$reload" != "" ]; then - echo "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname - echo "Reloaded $hostname region(s)" - fi -fi - -if [ $HBASE_BALANCER_STATE != "false" ]; then - echo "Restoring balancer state to" $HBASE_BALANCER_STATE - echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null -fi - -# Cleanup tmp files. -trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase deleted file mode 100755 index 13c265aea4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase +++ /dev/null @@ -1,373 +0,0 @@ -#! /usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# The hbase command script. Based on the hadoop command script putting -# in hbase classes, libs and configurations ahead of hadoop's. -# -# TODO: Narrow the amount of duplicated code. -# -# Environment Variables: -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HBASE_CLASSPATH Extra Java CLASSPATH entries. -# -# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding -# native libaries. -# -# HBASE_OPTS Extra Java runtime options. -# -# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -# -# HBASE_ROOT_LOGGER The root appender. Default is INFO,console -# -# MAVEN_HOME Where mvn is installed. -# -# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. -# Defaults to the jar packaged with HBase. -# -# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -# Empty by default. -# -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -# Detect if we are in hbase sources dir -in_dev_env=false -if [ -d "${HBASE_HOME}/target" ]; then - in_dev_env=true -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hbase " - echo "where an option from one of these categories:" - echo "" - echo "DBA TOOLS" - echo " shell run the HBase shell" - echo " hbck run the hbase 'fsck' tool" - echo " hlog write-ahead-log analyzer" - echo " hfile store file analyzer" - echo " zkcli run the ZooKeeper shell" - echo "" - echo "PROCESS MANAGEMENT" - echo " master run an HBase HMaster node" - echo " regionserver run an HBase HRegionServer node" - echo " zookeeper run a Zookeeper server" - echo " rest run an HBase REST server" - echo " thrift run the HBase Thrift server" - echo " thrift2 run the HBase Thrift2 server" - echo " avro run an HBase Avro server" - echo "" - echo "PACKAGE MANAGEMENT" - echo " classpath dump hbase CLASSPATH" - echo " version print the version" - echo "" - echo " or" - echo " CLASSNAME run the class named CLASSNAME" - echo "Most commands print help when invoked w/o parameters." - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -MVN="mvn" -if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn -fi - -# override default settings for this command, if applicable -if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then - . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" -fi - -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX -fi - -# so that filenames w/ spaces are handled correctly in loops below -IFS= - -# CLASSPATH initially contains $HBASE_CONF_DIR -CLASSPATH="${HBASE_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -add_maven_deps_to_classpath() { - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - # ------------------- - # Titan build support - # ------------------- - # Check for Maven command line options file. - # If titan-hbase has already been built, then this file will contain options - # to target the version of Hadoop used during compilation. This is - # necessary when the user compiles titan-hbase with a non-default Hadoop - # version (e.g. 2.2.0). If we omitted this check, then we'd try to start - # HBase with the default Hadoop version (currentl 1.0.4), and it would break. - MVN_OPTS= - optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" - unset IFS - if [ -f "${optf}" ] - then - MVN_OPTS=`cat "${optf}"` - fi - ${MVN} -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -add_maven_main_classes_to_classpath() { - if [ -d "$HBASE_HOME/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME/target/classes - fi -} - -add_maven_test_classes_to_classpath() { - # For developers, add hbase classes to CLASSPATH - f="$HBASE_HOME/target/test-classes" - if [ -d "${f}" ]; then - CLASSPATH=${CLASSPATH}:${f} - fi -} - -# Add maven target directory -if $in_dev_env; then - add_maven_deps_to_classpath - add_maven_main_classes_to_classpath - add_maven_test_classes_to_classpath -fi - -# For releases, add hbase & webapps to CLASSPATH -# Webapps must come first else it messes up Jetty -if [ -d "$HBASE_HOME/hbase-webapps" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME -fi -if [ -d "$HBASE_HOME/target/hbase-webapps" ]; then - CLASSPATH="${CLASSPATH}:${HBASE_HOME}/target" -fi -for f in $HBASE_HOME/hbase*.jar; do - if [[ $f = *sources.jar ]] - then - : # Skip sources.jar - elif [ -f $f ] - then - CLASSPATH=${CLASSPATH}:$f; - fi -done - -# Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# Add user-specified CLASSPATH first -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${HBASE_CLASSPATH}:${CLASSPATH} -fi - -# default log directory & file -if [ "$HBASE_LOG_DIR" = "" ]; then - HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -if [ "$HBASE_LOGFILE" = "" ]; then - HBASE_LOGFILE='hbase.log' -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - -function append_path() { - if [ -z "$1" ]; then - echo $2 - else - echo $1:$2 - fi -} - -JAVA_PLATFORM="" - -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) -if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) -fi - -if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then - if [ -z $JAVA_PLATFORM ]; then - JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - fi - if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) - fi - - if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) - fi -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -# restore ordinary behaviour -unset IFS - -#Set the right GC options based on the what we are running -declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") -for cmd in ${server_cmds[@]}; do - if [[ $cmd == $COMMAND ]]; then - server=true - break - fi -done - -if [[ $server ]]; then - HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" -else - HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" -fi - -# figure out which class to run -if [ "$COMMAND" = "shell" ] ; then - # eg export JRUBY_HOME=/usr/local/share/jruby - if [ "$JRUBY_HOME" != "" ] ; then - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" -elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -elif [ "$COMMAND" = "hlog" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' -elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFile' -elif [ "$COMMAND" = "zkcli" ] ; then - # ZooKeeperMainServerArg returns '-server HOST:PORT' or empty string. - SERVER_ARG=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServerArg` - CLASS="org.apache.zookeeper.ZooKeeperMain ${SERVER_ARG}" - -elif [ "$COMMAND" = "master" ] ; then - CLASS='org.apache.hadoop.hbase.master.HMaster' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" - fi -elif [ "$COMMAND" = "regionserver" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" - fi -elif [ "$COMMAND" = "thrift" ] ; then - CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "thrift2" ] ; then - CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "rest" ] ; then - CLASS='org.apache.hadoop.hbase.rest.Main' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" - fi -elif [ "$COMMAND" = "avro" ] ; then - CLASS='org.apache.hadoop.hbase.avro.AvroServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_AVRO_OPTS" - fi -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" - fi - -elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit 0 -elif [ "$COMMAND" = "version" ] ; then - CLASS='org.apache.hadoop.hbase.util.VersionInfo' -else - CLASS=$COMMAND -fi - -# Have JVM dump heap if we run out of memory. Files will be 'launch directory' -# and are named like the following: java_pid21612.hprof. Apparently it doesn't -# 'cost' to have this flag enabled. Its a 1.6 flag only. See: -# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" -HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" -HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" -HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" -fi - -# Enable security logging on the master and regionserver only -if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}" -else - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" -fi - -# Exec unless HBASE_NOEXEC is set. -if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -else - exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh deleted file mode 100755 index 966744ae52..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh +++ /dev/null @@ -1,120 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# included in all the hbase scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* -# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -# resolve links - "${BASH_SOURCE-$0}" may be a softlink - -this="${BASH_SOURCE-$0}" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin">/dev/null; pwd` -this="$bin/$script" - -# the root of the hbase installation -if [ -z "$HBASE_HOME" ]; then - export HBASE_HOME=`dirname "$this"`/.. -fi - -#check to see if the conf dir or hbase home are given as an optional arguments -while [ $# -gt 1 ] -do - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HBASE_CONF_DIR=$confdir - elif [ "--hosts" = "$1" ] - then - shift - hosts=$1 - shift - HBASE_REGIONSERVERS=$hosts - else - # Presume we are at end of options and break - break - fi -done - -# Allow alternate hbase conf dir location. -HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" -# List of hbase regions servers. -HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" -# List of hbase secondary masters. -HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" - -# Source the hbase-env.sh. Will have JAVA_HOME defined. -# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. -if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then - . "${HBASE_CONF_DIR}/hbase-env.sh" - export HBASE_ENV_INIT="true" -fi - -# Newer versions of glibc use an arena memory allocator that causes virtual -# memory usage to explode. Tune the variable down to prevent vmem explosion. -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} - -if [ -z "$JAVA_HOME" ]; then - for candidate in \ - /usr/lib/jvm/java-6-sun \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ - /usr/lib/j2sdk1.6-sun \ - /usr/java/jdk1.6* \ - /usr/java/jre1.6* \ - /Library/Java/Home ; do - if [ -e $candidate/bin/java ]; then - export JAVA_HOME=$candidate - break - fi - done - # if we didn't set it - if [ -z "$JAVA_HOME" ]; then - cat 1>&2 < http://java.sun.com/javase/downloads/ < | -| | -| HBase requires Java 1.6 or later. | -| NOTE: This script will find Sun Java whether you install using the | -| binary or the RPM based installer. | -+======================================================================+ -EOF - exit 1 - fi -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh deleted file mode 100755 index 6f680905cd..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Runs a Hadoop hbase command as a daemon. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_LOG_DIR Where log files are stored. PWD by default. -# HBASE_PID_DIR The pid files are stored. /tmp by default. -# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default -# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. -# -# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh - -usage="Usage: hbase-daemon.sh [--config ]\ - (start|stop|restart) \ - " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# get arguments -startStop=$1 -shift - -command=$1 -shift - -hbase_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" - num=$prev - done - mv -f "$log" "$log.$num"; - fi -} - -wait_until_done () -{ - p=$1 - cnt=${HBASE_SLAVE_TIMEOUT:-300} - origcnt=$cnt - while kill -0 $p > /dev/null 2>&1; do - if [ $cnt -gt 1 ]; then - cnt=`expr $cnt - 1` - sleep 1 - else - echo "Process did not complete after $origcnt seconds, killing." - kill -9 $p - exit 1 - fi - done - return 0 -} - -# get log directory -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_PID_DIR" = "" ]; then - HBASE_PID_DIR=/tmp -fi - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -# Some variables -# Work out java location so can print version into log. -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi -JAVA=$JAVA_HOME/bin/java -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -export HBASE_ROOT_LOGGER="INFO,DRFA" -export HBASE_SECURITY_LOGGER="INFO,DRFAS" -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid - -if [ -n "$SERVER_GC_OPTS" ]; then - export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi -if [ -n "$CLIENT_GC_OPTS" ]; then - export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi - -# Set default scheduling priority -if [ "$HBASE_NICENESS" = "" ]; then - export HBASE_NICENESS=0 -fi - -case $startStop in - - (start) - mkdir -p "$HBASE_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - hbase_rotate_log $logout - hbase_rotate_log $loggc - echo starting $command, logging to $logout - # Add to the command log file vital stats on our environment. - echo "`date` Starting $command on `hostname`" >> $loglog - echo "`ulimit -a`" >> $loglog 2>&1 - nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - $command "$@" $startStop > "$logout" 2>&1 < /dev/null & - echo $! > $pid - sleep 1; head "$logout" - ;; - - (stop) - if [ -f $pid ]; then - # kill -0 == see if the PID exists - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo -n stopping $command - echo "`date` Terminating $command" >> $loglog - kill `cat $pid` > /dev/null 2>&1 - while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; - done - rm $pid - echo - else - retval=$? - echo no $command to stop because kill -0 of pid `cat $pid` failed with status $retval - fi - else - echo no $command to stop because no pid file $pid - fi - ;; - - (restart) - thiscmd=$0 - args=$@ - # stop the command - $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & - wait_until_done $! - # wait a user-specified sleep period - sp=${HBASE_RESTART_SLEEP:-3} - if [ $sp -gt 0 ]; then - sleep $sp - fi - # start the command - $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & - wait_until_done $! - ;; - - (*) - echo $usage - exit 1 - ;; - -esac diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb deleted file mode 100755 index c7925150cb..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb +++ /dev/null @@ -1,185 +0,0 @@ -# -# Copyright 2009 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports -# and hbase commands and then loads jirb. Outputs a banner that tells user -# where to find help, shell version, and loads up a custom hirb. - -# TODO: Add 'debug' support (client-side logs show in shell). Add it as -# command-line option and as command. -# TODO: Interrupt a table creation or a connection to a bad master. Currently -# has to time out. Below we've set down the retries for rpc and hbase but -# still can be annoying (And there seem to be times when we'll retry for -# ever regardless) -# TODO: Add support for listing and manipulating catalog tables, etc. -# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes - -# Run the java magic include and import basic HBase types that will help ease -# hbase hacking. -include Java - -# Some goodies for hirb. Should these be left up to the user's discretion? -require 'irb/completion' - -# Add HBase to the ruby load path -if File.exists?(File.join(File.dirname(__FILE__), "ruby", "hbase.rb")) - $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "ruby") -end - -# -# FIXME: Switch args processing to getopt -# -# See if there are args for this shell. If any, read and then strip from ARGV -# so they don't go through to irb. Output shell 'usage' if user types '--help' -cmdline_help = </dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1" - exit -fi - -# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] -export HBASE_MASTER_OPTS=" " - -run_master () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_MASTER_ARGS="\ - -D hbase.master.port=`expr 60000 + $DN` \ - -D hbase.master.info.port=`expr 60010 + $DN` \ - --backup" - "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_master $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh deleted file mode 100755 index a4d5a1d932..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple regionservers on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 100 regionservers (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1 2" - exit -fi - -# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] -export HBASE_REGIONSERVER_OPTS=" " - -run_regionserver () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 60200 + $DN` \ - -D hbase.regionserver.info.port=`expr 60300 + $DN`" - "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_regionserver $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb deleted file mode 100755 index 5c6bea0ae1..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb +++ /dev/null @@ -1,475 +0,0 @@ -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Moves regions. Will confirm region access in current location and will -# not move a new region until successful confirm of region loading in new -# location. Presumes balancer is disabled when we run (not harmful if its -# on but this script and balancer will end up fighting each other). -# Does not work for case of multiple regionservers all running on the -# one node. -require 'optparse' -include Java -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.Get -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.HConnectionManager -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; -import org.apache.hadoop.hbase.HServerAddress -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.util.Writables -import org.apache.hadoop.conf.Configuration -import org.apache.commons.logging.Log -import org.apache.commons.logging.LogFactory - -# Name of this script -NAME = "region_mover" - -# Get root table reference -def getRootTable(config) - # Keep meta reference in ruby global - if not $ROOT - $ROOT = HTable.new(config, HConstants::ROOT_TABLE_NAME) - end - return $ROOT -end - -# Get meta table reference -def getMetaTable(config) - # Keep meta reference in ruby global - if not $META - $META = HTable.new(config, HConstants::META_TABLE_NAME) - end - return $META -end - -# Get table instance. -# Maintains cache of table instances. -def getTable(config, name) - # Keep dictionary of tables in ruby global - if not $TABLES - $TABLES = {} - end - key = Bytes.toString(name) - if not $TABLES[key] - $TABLES[key] = HTable.new(config, name) - end - return $TABLES[key] -end - - -# Returns true if passed region is still on 'original' when we look at .META. -def isSameServer(admin, r, original) - server = getServerNameForRegion(admin, r) - return false unless server - return true unless original - return server == original -end - -class RubyAbortable - include org.apache.hadoop.hbase.Abortable - def abort(why, e) - puts "ABORTED! why=" + why + ", e=" + e.to_s - end -end - -# Get servername that is up in .META.; this is hostname + port + startcode comma-delimited. -# Can return nil -def getServerNameForRegion(admin, r) - if r.isRootRegion() - # Hack - tracker = org.apache.hadoop.hbase.zookeeper.RootRegionTracker.new(admin.getConnection().getZooKeeperWatcher(), RubyAbortable.new()) - tracker.start() - while not tracker.isLocationAvailable() - sleep 0.1 - end - # Make a fake servername by appending ',' - rootServer = tracker.getRootRegionLocation().toString() + "," - tracker.stop() - return rootServer - end - table = nil - if r.isMetaRegion() - table = getRootTable(admin.getConfiguration()) - else - table = getMetaTable(admin.getConfiguration()) - end - g = Get.new(r.getRegionName()) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - result = table.get(g) - server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - return nil unless server - return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s -end - -# Trys to scan a row from passed region -# Throws exception if can't -def isSuccessfulScan(admin, r) - scan = Scan.new(r.getStartKey()) - scan.setBatch(1) - scan.setCaching(1) - scan.setFilter(FirstKeyOnlyFilter.new()) - table = getTable(admin.getConfiguration(), r.getTableName()) - scanner = table.getScanner(scan) - begin - results = scanner.next() - # We might scan into next region, this might be an empty table. - # But if no exception, presume scanning is working. - ensure - scanner.close() - table.close() - end -end - -# Check region has moved successful and is indeed hosted on another server -# Wait until that is the case. -def move(admin, r, newServer, original) - # Now move it. Do it in a loop so can retry if fail. Have seen issue where - # we tried move region but failed and retry put it back on old location; - # retry in this case. - retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) - count = 0 - same = true - while count < retries and same - if count > 0 - $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s) - end - count = count + 1 - begin - admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) - rescue java.lang.reflect.UndeclaredThrowableException => e - $LOG.info("Exception moving " + r.getEncodedName() + - "; split/moved? Continuing: " + e) - return - end - # Wait till its up on new server before moving on - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.move.wait.max", 60) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - same = isSameServer(admin, r, original) - break unless same - sleep 0.1 - end - end - raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same - # Assert can Scan from new location. - isSuccessfulScan(admin, r) -end - -# Return the hostname portion of a servername (all up to first ',') -def getHostnamePortFromServerName(serverName) - parts = serverName.split(',') - return parts[0] + ":" + parts[1] -end - -# Return the hostname:port out of a servername (all up to first ',') -def getHostnameFromServerName(serverName) - return serverName.split(',')[0] -end - -# Return array of servernames where servername is hostname+port+startcode -# comma-delimited -def getServers(admin) - serverInfos = admin.getClusterStatus().getServerInfo() - servers = [] - for server in serverInfos - servers << server.getServerName() - end - return servers -end - -# Remove the servername whose hostname portion matches from the passed -# array of servers. Returns as side-effect the servername removed. -def stripServer(servers, hostname) - count = servers.length - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = servers.delete(server) - end - end - # Check server to exclude is actually present - raise RuntimeError, "Server %s not online" % hostname unless servers.length < count - return servername -end - -# Returns a new serverlist that excludes the servername whose hostname portion -# matches from the passed array of servers. -def stripExcludes(servers, excludefile) - excludes = readExcludes(excludefile) - servers = servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) } - # return updated servers list - return servers -end - - -# Return servername that matches passed hostname -def getServerName(servers, hostname) - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = server - break - end - end - raise ArgumentError, "Server %s not online" % hostname unless servername - return servername -end - -# Create a logger and disable the DEBUG-level annoying client logging -def configureLogging(options) - apacheLogger = LogFactory.getLog(NAME) - # Configure log4j to not spew so much - unless (options[:debug]) - logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase.client") - logger.setLevel(org.apache.log4j.Level::INFO) - end - return apacheLogger -end - -# Get configuration instance -def getConfiguration() - config = HBaseConfiguration.create() - # No prefetching on .META. - config.setInt("hbase.client.prefetch.limit", 1) - # Make a config that retries at short intervals many times - config.setInt("hbase.client.pause", 500) - config.setInt("hbase.client.retries.number", 100) - return config -end - -# Now get list of regions on targetServer -def getRegions(config, servername) - connection = HConnectionManager::getConnection(config) - parts = servername.split(',') - rs = connection.getHRegionConnection(parts[0], parts[1].to_i) - return rs.getOnlineRegions() -end - -def deleteFile(filename) - f = java.io.File.new(filename) - f.delete() if f.exists() -end - -# Write HRegionInfo to file -# Need to serialize in case non-printable characters. -# Format is count of regionnames followed by serialized regionnames. -def writeFile(filename, regions) - fos = java.io.FileOutputStream.new(filename) - dos = java.io.DataOutputStream.new(fos) - # Write out a count of region names - dos.writeInt(regions.size()) - # Write actual region names. - for r in regions - bytes = Writables.getBytes(r) - Bytes.writeByteArray(dos, bytes) - end - dos.close() -end - -# See writeFile above. -# Returns array of HRegionInfos -def readFile(filename) - f = java.io.File.new(filename) - return java.util.ArrayList.new() unless f.exists() - fis = java.io.FileInputStream.new(f) - dis = java.io.DataInputStream.new(fis) - # Read count of regions - count = dis.readInt() - regions = java.util.ArrayList.new(count) - index = 0 - while index < count - regions.add(Writables.getHRegionInfo(Bytes.readByteArray(dis))) - index = index + 1 - end - dis.close() - return regions -end - -# Move regions off the passed hostname -def unloadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Clean up any old files. - filename = getFilename(options, hostname) - deleteFile(filename) - # Get an admin instance - admin = HBaseAdmin.new(config) - servers = getServers(admin) - # Remove the server we are unloading from from list of servers. - # Side-effect is the servername that matches this hostname - servername = stripServer(servers, hostname) - - # Remove the servers in our exclude list from list of servers. - servers = stripExcludes(servers, options[:excludesFile]) - puts "Valid region move targets: ", servers - movedRegions = java.util.ArrayList.new() - while true - rs = getRegions(config, servername) - break if rs.length == 0 - count = 0 - $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + - " during this cycle"); - for r in rs - # Get a random server to move the region to. - server = servers[rand(servers.length)] - $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + - " of " + rs.length.to_s + ") to server=" + server); - count = count + 1 - # Assert we can scan region in its current location - isSuccessfulScan(admin, r) - # Now move it. - move(admin, r, server, servername) - movedRegions.add(r) - end - end - if movedRegions.size() > 0 - # Write out file of regions moved - writeFile(filename, movedRegions) - $LOG.info("Wrote list of moved regions to " + filename) - end -end - -# Move regions to the passed hostname -def loadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Get an admin instance - admin = HBaseAdmin.new(config) - filename = getFilename(options, hostname) - regions = readFile(filename) - return if regions.isEmpty() - servername = nil - # Wait till server is up - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.serverstart.wait.max", 180) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - servers = getServers(admin) - begin - servername = getServerName(servers, hostname) - rescue ArgumentError => e - $LOG.info("hostname=" + hostname.to_s + " is not up yet, waiting"); - end - break if servername - sleep 0.5 - end - $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) - count = 0 - for r in regions - exists = false - begin - isSuccessfulScan(admin, r) - exists = true - rescue org.apache.hadoop.hbase.NotServingRegionException => e - $LOG.info("Failed scan of " + e.message) - end - count = count + 1 - next unless exists - currentServer = getServerNameForRegion(admin, r) - if currentServer and currentServer == servername - $LOG.info("Region " + r.getRegionNameAsString() + " (" + count.to_s + - " of " + regions.length.to_s + ") already on target server=" + servername) - next - end - $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + - " of " + regions.length.to_s + ") to server=" + servername); - move(admin, r, servername, currentServer) - end -end - -# Returns an array of hosts to exclude as region move targets -def readExcludes(filename) - if filename == nil - return java.util.ArrayList.new() - end - if ! File.exist?(filename) - puts "Error: Unable to read host exclude file: ", filename - raise RuntimeError - end - - f = File.new(filename, "r") - # Read excluded hosts list - excludes = java.util.ArrayList.new() - while (line = f.gets) - line.strip! # do an inplace drop of pre and post whitespaces - excludes.add(line) unless line.empty? # exclude empty lines - end - puts "Excluding hosts as region move targets: ", excludes - f.close - - return excludes -end - -def getFilename(options, targetServer) - filename = options[:file] - if not filename - filename = "/tmp/" + targetServer - end - return filename -end - - -# Do command-line parsing -options = {} -optparse = OptionParser.new do |opts| - opts.banner = "Usage: #{NAME}.rb [options] load|unload " - opts.separator 'Load or unload regions by moving one at a time' - options[:file] = nil - opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| - options[:file] = file - end - opts.on('-h', '--help', 'Display usage information') do - puts opts - exit - end - options[:debug] = false - opts.on('-d', '--debug', 'Display extra debug logging') do - options[:debug] = true - end - opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| - options[:excludesFile] = file - end -end -optparse.parse! - -# Check ARGVs -if ARGV.length < 2 - puts optparse - exit 1 -end -hostname = ARGV[1] -if not hostname - opts optparse - exit 2 -end -# Create a logger and save it to ruby global -$LOG = configureLogging(options) -case ARGV[0] - when 'load' - loadRegions(options, hostname) - when 'unload' - unloadRegions(options, hostname) - else - puts optparse - exit 3 -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb deleted file mode 100755 index 5ccc8c2e6d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# View the current status of all regions on an HBase cluster. This is -# predominantly used to determined if all the regions in META have been -# onlined yet on startup. -# -# To use this script, run: -# -# ${HBASE_HOME}/bin/hbase org.jruby.Main region_status.rb [wait] [--table ] - - -require 'optparse' - -usage = 'Usage : ./hbase org.jruby.Main region_status.rb [wait]' + - '[--table ]\n' -OptionParser.new do |o| - o.banner = usage - o.on('-t', '--table TABLENAME', 'Only process TABLENAME') do |tablename| - $tablename = tablename - end - o.on('-h', '--help', 'Display help message') { puts o; exit } - o.parse! -end - -SHOULD_WAIT = ARGV[0] == 'wait' -if ARGV[0] and not SHOULD_WAIT - print usage - exit 1 -end - - -require 'java' - -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.MasterNotRunningException -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.util.Writables - -# disable debug logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) -org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) - -config = HBaseConfiguration.create -config.set 'fs.default.name', config.get(HConstants::HBASE_DIR) - -# wait until the master is running -admin = nil -while true - begin - admin = HBaseAdmin.new config - break - rescue MasterNotRunningException => e - print 'Waiting for master to start...\n' - sleep 1 - end -end - -meta_count = 0 -server_count = 0 - -# scan META to see how many regions we should have -if $tablename.nil? - scan = Scan.new -else - tableNameMetaPrefix = $tablename + HConstants::META_ROW_DELIMITER.chr - scan = Scan.new( - (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes - ) -end -scan.cache_blocks = false -scan.caching = 10 -scan.setFilter(FirstKeyOnlyFilter.new) -INFO = 'info'.to_java_bytes -REGION_INFO = 'regioninfo'.to_java_bytes -scan.addColumn INFO, REGION_INFO -table = nil -iter = nil -while true - begin - table = HTable.new config, '.META.'.to_java_bytes - scanner = table.getScanner(scan) - iter = scanner.iterator - break - rescue IOException => ioe - print "Exception trying to scan META: #{ioe}" - sleep 1 - end -end -while iter.hasNext - result = iter.next - rowid = Bytes.toString(result.getRow()) - rowidStr = java.lang.String.new(rowid) - if not $tablename.nil? and not rowidStr.startsWith(tableNameMetaPrefix) - # Gone too far, break - break - end - region = Writables.getHRegionInfo result.getValue(INFO, REGION_INFO) - if not region.isOffline - # only include regions that should be online - meta_count += 1 - end -end -scanner.close -# If we're trying to see the status of all HBase tables, we need to include the -# -ROOT- & .META. tables, that are not included in our scan -if $tablename.nil? - meta_count += 2 -end - -# query the master to see how many regions are on region servers -if not $tablename.nil? - $tableq = HTable.new config, $tablename.to_java_bytes -end -while true - if $tablename.nil? - server_count = admin.getClusterStatus().getRegionsCount() - else - server_count = $tableq.getRegionsInfo().size() - end - print "Region Status: #{server_count} / #{meta_count}\n" - if SHOULD_WAIT and server_count < meta_count - #continue this loop until server & meta count match - sleep 10 - else - break - end -end - -exit server_count == meta_count ? 0 : 1 diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh deleted file mode 100755 index 23f8d325e8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -function usage() { - echo $usage - exit 1 -} - - -RR_RS=1 -RR_MASTER=1 -RR_GRACEFUL=0 - -for x in "$@" ; do - case "$x" in - --rs-only|-r) - RR_RS=1 - RR_MASTER=0 - RR_GRACEFUL=0 - ;; - --master-only) - RR_RS=0 - RR_MASTER=1 - RR_GRACEFUL=0 - ;; - --graceful) - RR_RS=0 - RR_MASTER=0 - RR_GRACEFUL=1 - ;; - *) - echo Bad argument: $x - usage - exit 1 - ;; - esac -done - -# quick function to get a value from the HBase config file -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'false' ]; then - if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then - echo Cant do selective rolling restart if not running distributed - exit 1 - fi - "$bin"/hbase-daemon.sh restart master -else - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - - if [ $RR_MASTER -eq 1 ]; then - # stop all masters before re-start to avoid races for master znode - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - echo -n "Waiting for Master ZNode ${zmaster} to expire" - while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do - echo -n "." - sleep 1 - done - echo #force a newline - - # all masters are down, now restart - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup - - echo "Wait a minute for master to come up join cluster" - sleep 60 - - # Master joing cluster will start in cleaning out regions in transition. - # Wait until the master has cleaned out regions in transition before - # giving it a bunch of work to do; master is vulnerable during startup - zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` - if [ "$zunassigned" == "null" ]; then zunassigned="unassigned"; fi - zunassigned="$zparent/$zunassigned" - echo -n "Waiting for ${zunassigned} to empty" - while true ; do - unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` - if test 0 -eq ${unassigned} - then - break - else - echo -n " ${unassigned}" - fi - sleep 1 - done - fi - - if [ $RR_RS -eq 1 ]; then - # unlike the masters, roll all regionservers one-at-a-time - export HBASE_SLAVE_PARALLEL=false - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" restart regionserver - fi - - if [ $RR_GRACEFUL -eq 1 ]; then - # gracefully restart all online regionservers - zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zkrs" == "null" ]; then zkrs="rs"; fi - zkrs="$zparent/$zkrs" - online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - for rs in $online_regionservers - do - rs_parts=(${rs//,/ }) - hostname=${rs_parts[0]} - echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" - sleep 1 - done - fi -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb deleted file mode 100644 index 4d97cd0948..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# HBase ruby classes. -# Has wrapper classes for org.apache.hadoop.hbase.client.HBaseAdmin -# and for org.apache.hadoop.hbase.client.HTable. Classes take -# Formatters on construction and outputs any results using -# Formatter methods. These classes are only really for use by -# the hirb.rb HBase Shell script; they don't make much sense elsewhere. -# For example, the exists method on Admin class prints to the formatter -# whether the table exists and returns nil regardless. -include Java - -include_class('java.lang.Integer') {|package,name| "J#{name}" } -include_class('java.lang.Long') {|package,name| "J#{name}" } -include_class('java.lang.Boolean') {|package,name| "J#{name}" } - -module HBaseConstants - COLUMN = "COLUMN" - COLUMNS = "COLUMNS" - TIMESTAMP = "TIMESTAMP" - TIMERANGE = "TIMERANGE" - NAME = org.apache.hadoop.hbase.HConstants::NAME - VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS - IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY - CONFIG = org.apache.hadoop.hbase.HConstants::CONFIG - STOPROW = "STOPROW" - STARTROW = "STARTROW" - ENDROW = STOPROW - RAW = "RAW" - LIMIT = "LIMIT" - METHOD = "METHOD" - MAXLENGTH = "MAXLENGTH" - CACHE_BLOCKS = "CACHE_BLOCKS" - REPLICATION_SCOPE = "REPLICATION_SCOPE" - INTERVAL = 'INTERVAL' - CACHE = 'CACHE' - FILTER = 'FILTER' - SPLITS = 'SPLITS' - SPLITS_FILE = 'SPLITS_FILE' - SPLITALGO = 'SPLITALGO' - NUMREGIONS = 'NUMREGIONS' - - # Load constants from hbase java API - def self.promote_constants(constants) - # The constants to import are all in uppercase - constants.each do |c| - next if c =~ /DEFAULT_.*/ || c != c.upcase - next if eval("defined?(#{c})") - eval("#{c} = '#{c}'") - end - end - - promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) - promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) -end - -# Include classes definition -require 'hbase/hbase' -require 'hbase/admin' -require 'hbase/table' -require 'hbase/replication_admin' -require 'hbase/security' diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb deleted file mode 100644 index 4ad3d88c74..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb +++ /dev/null @@ -1,667 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java -java_import org.apache.hadoop.hbase.util.Pair -java_import org.apache.hadoop.hbase.util.RegionSplitter - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class Admin - include HBaseConstants - - def initialize(configuration, formatter) - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - connection = @admin.getConnection() - @conf = configuration - @zk_wrapper = connection.getZooKeeperWatcher() - zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() - @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of tables in hbase - def list(regex = ".*") - begin - # Use the old listTables API first for compatibility with older servers - @admin.listTables(regex).map { |t| t.getNameAsString } - rescue => e - # listTables failed, try the new unprivileged getTableNames API if the cause was - # an AccessDeniedException - if e.cause.kind_of? org.apache.hadoop.ipc.RemoteException and e.cause.unwrapRemoteException().kind_of? org.apache.hadoop.hbase.security.AccessDeniedException - @admin.getTableNames(regex) - else - # Not an access control failure, re-raise - raise e - end - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region flush - def flush(table_or_region_name) - @admin.flush(table_or_region_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family compaction - def compact(table_or_region_name, family = nil) - if family == nil - @admin.compact(table_or_region_name) - else - # We are compacting a column family within a region. - @admin.compact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family major compaction - def major_compact(table_or_region_name, family = nil) - if family == nil - @admin.majorCompact(table_or_region_name) - else - # We are major compacting a column family within a region or table. - @admin.majorCompact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a regionserver's HLog roll - def hlog_roll(server_name) - @admin.rollHLogWriter(server_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region split - def split(table_or_region_name, split_point) - if split_point == nil - @admin.split(table_or_region_name) - else - @admin.split(table_or_region_name, split_point) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a cluster balance - # Returns true if balancer ran - def balancer() - @admin.balancer() - end - - #---------------------------------------------------------------------------------------------- - # Enable/disable balancer - # Returns previous balancer switch setting. - def balance_switch(enableDisable) - @admin.setBalancerRunning( - java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false)) - end - - #---------------------------------------------------------------------------------------------- - # Enables a table - def enable(table_name) - tableExists(table_name) - return if enabled?(table_name) - @admin.enableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Enables all tables matching the given regex - def enable_all(regex) - regex = regex.to_s - @admin.enableTables(regex) - end - - #---------------------------------------------------------------------------------------------- - # Disables a table - def disable(table_name) - tableExists(table_name) - return if disabled?(table_name) - @admin.disableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Disables all tables matching the given regex - def disable_all(regex) - regex = regex.to_s - @admin.disableTables(regex).map { |t| t.getNameAsString } - end - - #--------------------------------------------------------------------------------------------- - # Throw exception if table doesn't exist - def tableExists(table_name) - raise ArgumentError, "Table #{table_name} does not exist.'" unless exists?(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table disabled? - def disabled?(table_name) - @admin.isTableDisabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop(table_name) - tableExists(table_name) - raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name) - - @admin.deleteTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop_all(regex) - regex = regex.to_s - failed = @admin.deleteTables(regex).map { |t| t.getNameAsString } - return failed - end - - #---------------------------------------------------------------------------------------------- - # Returns ZooKeeper status dump - def zk_dump - org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) - end - - #---------------------------------------------------------------------------------------------- - # Creates a table - def create(table_name, *args) - # Fail if table name is not a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Flatten params array - args = args.flatten.compact - - # Fail if no column families defined - raise(ArgumentError, "Table must have at least one column family") if args.empty? - - # Start defining the table - htd = org.apache.hadoop.hbase.HTableDescriptor.new(table_name) - splits = nil - # Args are either columns or splits, add them to the table definition - # TODO: add table options support - args.each do |arg| - unless arg.kind_of?(String) || arg.kind_of?(Hash) - raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") - end - - if arg.kind_of?(String) - # the arg is a string, default action is to add a column to the table - htd.addFamily(hcd(arg, htd)) - else - # arg is a hash. 4 possibilities: - if (arg.has_key?(SPLITS) or arg.has_key?(SPLITS_FILE)) - if arg.has_key?(SPLITS_FILE) - unless File.exist?(arg[SPLITS_FILE]) - raise(ArgumentError, "Splits file #{arg[SPLITS_FILE]} doesn't exist") - end - arg[SPLITS] = [] - File.foreach(arg[SPLITS_FILE]) do |line| - arg[SPLITS].push(line.strip()) - end - end - - splits = Java::byte[][arg[SPLITS].size].new - idx = 0 - arg[SPLITS].each do |split| - splits[idx] = split.to_java_bytes - idx = idx + 1 - end - elsif (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO)) - # (1) deprecated region pre-split API - raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME) - raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS) - raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO) - raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 - num_regions = arg[NUMREGIONS] - split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) - splits = split_algo.split(JInteger.valueOf(num_regions)) - elsif (method = arg.delete(METHOD)) - # (2) table_att modification - raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' - raise(ArgumentError, "NUMREGIONS & SPLITALGO must both be specified") unless arg.has_key?(NUMREGIONS) == arg.has_key?(split_algo) - htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] - htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] - htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] - if arg[NUMREGIONS] - raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 - num_regions = arg[NUMREGIONS] - split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) - splits = split_algo.split(JInteger.valueOf(num_regions)) - end - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - htd.setValue(k, v) - end - end - else - # (3) column family spec - descriptor = hcd(arg, htd) - htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] - htd.addFamily(hcd(arg, htd)) - end - end - end - - if splits.nil? - # Perform the create table call - @admin.createTable(htd) - else - # Perform the create table call - @admin.createTable(htd, splits) - end - end - - #---------------------------------------------------------------------------------------------- - # Closes a region. - # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). - # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) - def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) - @admin.closeRegion(region_name, server) - end - end - - #---------------------------------------------------------------------------------------------- - #---------------------------------------------------------------------------------------------- - # Assign a region - def assign(region_name) - @admin.assign(region_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Unassign a region - def unassign(region_name, force) - @admin.unassign(region_name.to_java_bytes, java.lang.Boolean::valueOf(force)) - end - - #---------------------------------------------------------------------------------------------- - # Move a region - def move(encoded_region_name, server = nil) - @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) - end - - #---------------------------------------------------------------------------------------------- - # Returns table's structure description - def describe(table_name) - @admin.getTableDescriptor(table_name.to_java_bytes).to_s - end - - #---------------------------------------------------------------------------------------------- - # Truncates table (deletes all records by recreating the table) - def truncate(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - table_description = h_table.getTableDescriptor() - raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) - yield 'Disabling table...' if block_given? - @admin.disableTable(table_name) - - yield 'Dropping table...' if block_given? - @admin.deleteTable(table_name) - - yield 'Creating table...' if block_given? - @admin.createTable(table_description) - end - - # Check the status of alter command (number of regions reopened) - def alter_status(table_name) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - status = Pair.new() - begin - status = @admin.getAlterStatus(table_name.to_java_bytes) - if status.getSecond() != 0 - puts "#{status.getSecond() - status.getFirst()}/#{status.getSecond()} regions updated." - else - puts "All regions updated." - end - sleep 1 - end while status != nil && status.getFirst() != 0 - puts "Done." - end - - #---------------------------------------------------------------------------------------------- - # Change table structure or table options - def alter(table_name, wait = true, *args) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - # There should be at least one argument - raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? - - # Get table descriptor - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - # Process all args - args.each do |arg| - # Normalize args to support column name only alter specs - arg = { NAME => arg } if arg.kind_of?(String) - - # Normalize args to support shortcut delete syntax - arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - - # No method parameter, try to use the args as a column definition - unless method = arg.delete(METHOD) - # Note that we handle owner here, and also below (see (2)) as part of the "METHOD => 'table_att'" table attributes. - # In other words, if OWNER is specified, then METHOD is set to table_att. - # alter 'tablename', {OWNER => 'username'} (that is, METHOD => 'table_att' is not specified). - if arg[OWNER] - htd.setOwnerString(arg[OWNER]) - @admin.modifyTable(table_name.to_java_bytes, htd) - return - end - - descriptor = hcd(arg, htd) - - if arg[COMPRESSION_COMPACT] - descriptor.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) - end - column_name = descriptor.getNameAsString - - # If column already exist, then try to alter it. Create otherwise. - if htd.hasFamily(column_name.to_java_bytes) - @admin.modifyColumn(table_name, descriptor) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - else - @admin.addColumn(table_name, descriptor) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - end - next - end - - # Delete column family - if method == "delete" - raise(ArgumentError, "NAME parameter missing for delete method") unless arg[NAME] - @admin.deleteColumn(table_name, arg[NAME]) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - next - end - - # Change table attributes - if method == "table_att" - htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] - htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] - # (2) Here, we handle the alternate syntax of ownership setting, where method => 'table_att' is specified. - htd.setOwnerString(arg[OWNER]) if arg[OWNER] - - # set a coprocessor attribute - if arg.kind_of?(Hash) - arg.each do |key, value| - k = String.new(key) # prepare to strip - k.strip! - - if (k =~ /coprocessor/i) - # validate coprocessor specs - v = String.new(value) - v.strip! - if !(v =~ /^([^\|]*)\|([^\|]+)\|[\s]*([\d]*)[\s]*(\|.*)?$/) - raise ArgumentError, "Coprocessor value doesn't match spec: #{v}" - end - - # generate a coprocessor ordinal by checking max id of existing cps - maxId = 0 - htd.getValues().each do |k1, v1| - attrName = org.apache.hadoop.hbase.util.Bytes.toString(k1.get()) - # a cp key is coprocessor$(\d) - if (attrName =~ /coprocessor\$(\d+)/i) - ids = attrName.scan(/coprocessor\$(\d+)/i) - maxId = ids[0][0].to_i if ids[0][0].to_i > maxId - end - end - maxId += 1 - htd.setValue(k + "\$" + maxId.to_s, value) - end - end - end - - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - htd.setValue(k, v) - end - end - @admin.modifyTable(table_name.to_java_bytes, htd) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - next - end - - # Unset table attributes - if method == "table_att_unset" - if arg.kind_of?(Hash) - if (!arg[NAME]) - next - end - if (htd.getValue(arg[NAME]) == nil) - raise ArgumentError, "Can not find attribute: #{arg[NAME]}" - end - htd.remove(arg[NAME].to_java_bytes) - @admin.modifyTable(table_name.to_java_bytes, htd) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - end - next - end - - # Unknown method - raise ArgumentError, "Unknown method: #{method}" - end - end - - def status(format) - status = @admin.getClusterStatus() - if format == "detailed" - puts("version %s" % [ status.getHBaseVersion() ]) - # Put regions in transition first because usually empty - puts("%d regionsInTransition" % status.getRegionsInTransition().size()) - for k, v in status.getRegionsInTransition() - puts(" %s" % [v]) - end - master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors()) - if master_coprocs != nil - puts("master coprocessors: %s" % master_coprocs) - end - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - for name, region in status.getLoad(server).getRegionsLoad() - puts(" %s" % [ region.getNameAsString() ]) - puts(" %s" % [ region.toString() ]) - end - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - elsif format == "simple" - load = 0 - regions = 0 - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - load += status.getLoad(server).getNumberOfRequests() - regions += status.getLoad(server).getNumberOfRegions() - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - puts("Aggregate load: %d, regions: %d" % [ load , regions ] ) - else - puts "#{status.getServersSize} servers, #{status.getDeadServers} dead, #{'%.4f' % status.getAverageLoad} average load" - end - end - - #---------------------------------------------------------------------------------------------- - # - # Helper methods - # - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table enabled - def enabled?(table_name) - @admin.isTableEnabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - #Is supplied region name is encoded region name - def closeEncodedRegion?(region_name, server) - @admin.closeRegionWithEncodedRegionName(region_name, server) - end - - #---------------------------------------------------------------------------------------------- - # Return a new HColumnDescriptor made of passed args - def hcd(arg, htd) - # String arg, single parameter constructor - return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.kind_of?(String) - - raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg[NAME] - - family = htd.getFamily(name.to_java_bytes) - # create it if it's a new family - family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) - - family.setBlockCacheEnabled(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) - family.setScope(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) - family.setInMemory(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) - family.setTimeToLive(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::TTL])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) - family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) - family.setEncodeOnDisk(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK) - family.setBlocksize(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) - family.setMaxVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) - family.setMinVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) - bloomtype = arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER].upcase - unless org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else - family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.valueOf(bloomtype)) - end - end - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) - compression = arg[org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION].upcase - unless org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.include?(compression) - raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.join(" ")) - else - family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(compression)) - end - end - - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - family.setValue(k, v) - end - end - return family - end - - #---------------------------------------------------------------------------------------------- - # Enables/disables a region by name - def online(region_name, on_off) - # Open meta table - meta = org.apache.hadoop.hbase.client.HTable.new(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) - - # Read region info - # FIXME: fail gracefully if can't find the region - region_bytes = region_name.to_java_bytes - g = org.apache.hadoop.hbase.client.Get.new(region_bytes) - g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) - hri_bytes = meta.get(g).value - - # Change region status - hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) - hri.setOffline(on_off) - - # Write it back - put = org.apache.hadoop.hbase.client.Put.new(region_bytes) - put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri)) - meta.put(put) - end - - #---------------------------------------------------------------------------------------------- - # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Restore specified snapshot - def restore_snapshot(snapshot_name) - @admin.restoreSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Create a new table by cloning the snapshot content - def clone_snapshot(snapshot_name, table) - @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Delete specified snapshot - def delete_snapshot(snapshot_name) - @admin.deleteSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of snapshots - def list_snapshot - @admin.listSnapshots - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb deleted file mode 100644 index 2c37840dd2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -require 'hbase/admin' -require 'hbase/table' -require 'hbase/security' - -module Hbase - class Hbase - attr_accessor :configuration - - def initialize(config = nil) - # Create configuration - if config - self.configuration = config - else - self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create - # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. - configuration.setInt("hbase.client.retries.number", 7) - configuration.setInt("ipc.client.connect.max.retries", 3) - end - end - - def admin(formatter) - ::Hbase::Admin.new(configuration, formatter) - end - - # Create new one each time - def table(table, formatter) - ::Hbase::Table.new(configuration, table, formatter) - end - - def replication_admin(formatter) - ::Hbase::RepAdmin.new(configuration, formatter) - end - - def security_admin(formatter) - ::Hbase::SecurityAdmin.new(configuration, formatter) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb deleted file mode 100644 index 99f0479b0e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class RepAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @replication_admin = org.apache.hadoop.hbase.client.replication.ReplicationAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Add a new peer cluster to replicate to - def add_peer(id, cluster_key) - @replication_admin.addPeer(id, cluster_key) - end - - #---------------------------------------------------------------------------------------------- - # Remove a peer cluster, stops the replication - def remove_peer(id) - @replication_admin.removePeer(id) - end - - #--------------------------------------------------------------------------------------------- - # Show replcated tables/column families, and their ReplicationType - def list_replicated_tables - @replication_admin.listReplicated() - end - - #---------------------------------------------------------------------------------------------- - # List all peer clusters - def list_peers - @replication_admin.listPeers - end - - #---------------------------------------------------------------------------------------------- - # Get peer cluster state - def get_peer_state(id) - @replication_admin.getPeerState(id) - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication stream to the specified peer - def enable_peer(id) - @replication_admin.enablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Stop the replication stream to the specified peer - def disable_peer(id) - @replication_admin.disablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication, in an unknown state - def start_replication - @replication_admin.setReplicating(true) - end - - #---------------------------------------------------------------------------------------------- - # Kill switch for replication, stops all its features - def stop_replication - @replication_admin.setReplicating(false) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb deleted file mode 100644 index 5041ec4dbb..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb +++ /dev/null @@ -1,188 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class SecurityAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @config = configuration - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - # Verify that the specified permission is valid - if (permissions == nil || permissions.length == 0) - raise(ArgumentError, "Invalid permission: no actions associated with user") - end - - if (table_name != nil) - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - if (family != nil) - raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - # invoke cp endpoint to perform access controlse - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, table_name.to_java_bytes, - fambytes, qualbytes, permissions.to_java_bytes) - else - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, permissions.to_java_bytes) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - begin - protocol.grant(user_permission) - rescue java.io.IOException => e - if !(e.message.include? "java.lang.NoSuchMethodException") - raise e - end - - # Server has not the new API, try the old one - if (table_name == nil) - raise "Global permissions not supported by HBase Server" - end - - tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, permissions.to_java_bytes) - protocol.grant(user.to_java_bytes, tp) - end - end - - #---------------------------------------------------------------------------------------------- - def revoke(user, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - if (table_name != nil) - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - if (family != nil) - raise(ArgumentError, "Can't find family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - # invoke cp endpoint to perform access control - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, table_name.to_java_bytes, - fambytes, qualbytes, "".to_java_bytes) - else - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, "".to_java_bytes) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - begin - protocol.revoke(user_permission) - rescue java.io.IOException => e - if !(e.message.include? "java.lang.NoSuchMethodException") - raise e - end - - # Server has not the new API, try the old one - if (table_name == nil) - raise "Global permissions not supported by HBase Server" - end - - tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, "".to_java_bytes) - protocol.revoke(user.to_java_bytes, tp) - end - end - - #---------------------------------------------------------------------------------------------- - def user_permission(table_name=nil) - security_available? - - if (table_name != nil) - raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - perms = protocol.getUserPermissions(table_name != nil ? table_name.to_java_bytes : nil) - - res = {} - count = 0 - perms.each do |value| - user_name = String.from_java_bytes(value.getUser) - table = (value.getTable != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getTable) : '' - family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' - qualifier = (value.getQualifier != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : '' - - action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions - - if block_given? - yield(user_name, "#{table},#{family},#{qualifier}: #{action.to_s}") - else - res[user_name] ||= {} - res[user_name][family + ":" +qualifier] = action - end - count += 1 - end - - return ((block_given?) ? count : res) - end - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - # Make sure that security classes are available - def security_available?() - begin - org.apache.hadoop.hbase.security.access.AccessControllerProtocol - rescue NameError - raise(ArgumentError, "DISABLED: Security features are not available in this build of HBase") - end - end - - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb deleted file mode 100644 index ed8437ca88..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb +++ /dev/null @@ -1,366 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HTable - -module Hbase - class Table - include HBaseConstants - attr_reader :table - @@thread_pool = nil - - def initialize(configuration, table_name, formatter) - if @@thread_pool then - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name.to_java_bytes, @@thread_pool) - else - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name) - @@thread_pool = @table.getPool() - end - end - - #---------------------------------------------------------------------------------------------- - # Put a cell 'value' at specified table/row/column - def put(row, column, value, timestamp = nil) - p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) - family, qualifier = parse_column_name(column) - if timestamp - p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) - else - p.add(family, qualifier, value.to_s.to_java_bytes) - end - @table.put(p) - end - - #---------------------------------------------------------------------------------------------- - # Delete a cell - def delete(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - deleteall(row, column, timestamp) - end - - #---------------------------------------------------------------------------------------------- - # Delete a row - def deleteall(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp, nil) - if column - family, qualifier = parse_column_name(column) - d.deleteColumns(family, qualifier, timestamp) - end - @table.delete(d) - end - - #---------------------------------------------------------------------------------------------- - # Increment a counter atomically - def incr(row, column, value = nil) - value ||= 1 - family, qualifier = parse_column_name(column) - @table.incrementColumnValue(row.to_s.to_java_bytes, family, qualifier, value) - end - - #---------------------------------------------------------------------------------------------- - # Count rows in a table - def count(interval = 1000, caching_rows = 10) - # We can safely set scanner caching with the first key only filter - scan = org.apache.hadoop.hbase.client.Scan.new - scan.cache_blocks = false - scan.caching = caching_rows - scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) - - # Run the scanner - scanner = @table.getScanner(scan) - count = 0 - iter = scanner.iterator - - # Iterate results - while iter.hasNext - row = iter.next - count += 1 - next unless (block_given? && count % interval == 0) - # Allow command modules to visualize counting process - yield(count, - org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) - end - - # Return the counter - return count - end - - #---------------------------------------------------------------------------------------------- - # Get from table - def get(row, *args) - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - maxlength = -1 - - # Normalize args - args = args.first if args.first.kind_of?(Hash) - if args.kind_of?(String) || args.kind_of?(Array) - columns = [ args ].flatten.compact - args = { COLUMNS => columns } - end - - # - # Parse arguments - # - unless args.kind_of?(Hash) - raise ArgumentError, "Failed parse of of #{args.inspect}, #{args.class}" - end - - # Get maxlength parameter if passed - maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] - filter = args.delete(FILTER) if args[FILTER] - - unless args.empty? - columns = args[COLUMN] || args[COLUMNS] - if args[VERSIONS] - vers = args[VERSIONS] - else - vers = 1 - end - if columns - # Normalize types, convert string to an array of strings - columns = [ columns ] if columns.is_a?(String) - - # At this point it is either an array or some unsupported stuff - unless columns.kind_of?(Array) - raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" - end - - # Get each column name and add it to the filter - columns.each do |column| - family, qualifier = parse_column_name(column.to_s) - if qualifier - get.addColumn(family, qualifier) - else - get.addFamily(family) - end - end - - # Additional params - get.setMaxVersions(vers) - get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - else - # May have passed TIMESTAMP and row only; wants all columns from ts. - unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] - raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" - end - - get.setMaxVersions(vers) - # Set the timestamp/timerange - get.setTimeStamp(ts.to_i) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - end - end - - unless filter.class == String - get.setFilter(filter) - else - get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - # Call hbase for the results - result = @table.get(get) - return nil if result.isEmpty - - # Print out results. Result can be Cell or RowResult. - res = {} - result.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - value = to_string(column, kv, maxlength) - - if block_given? - yield(column, value) - else - res[column] = value - end - end - - # If block given, we've yielded all the results, otherwise just return them - return ((block_given?) ? nil : res) - end - - #---------------------------------------------------------------------------------------------- - # Fetches and decodes a counter value from hbase - def get_counter(row, column) - family, qualifier = parse_column_name(column.to_s) - # Format get request - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - get.addColumn(family, qualifier) - get.setMaxVersions(1) - - # Call hbase - result = @table.get(get) - return nil if result.isEmpty - - # Fetch cell value - cell = result.list[0] - org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) - end - - #---------------------------------------------------------------------------------------------- - # Scans whole table or a range of keys and returns rows matching specific criterias - def scan(args = {}) - unless args.kind_of?(Hash) - raise ArgumentError, "Arguments should be a hash. Failed to parse #{args.inspect}, #{args.class}" - end - - limit = args.delete("LIMIT") || -1 - maxlength = args.delete("MAXLENGTH") || -1 - - if args.any? - filter = args["FILTER"] - startrow = args["STARTROW"] || '' - stoprow = args["STOPROW"] - timestamp = args["TIMESTAMP"] - columns = args["COLUMNS"] || args["COLUMN"] || [] - cache_blocks = args["CACHE_BLOCKS"] || true - cache = args["CACHE"] || 0 - versions = args["VERSIONS"] || 1 - timerange = args[TIMERANGE] - raw = args["RAW"] || false - - # Normalize column names - columns = [columns] if columns.class == String - unless columns.kind_of?(Array) - raise ArgumentError.new("COLUMNS must be specified as a String or an Array") - end - - scan = if stoprow - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) - else - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) - end - - columns.each do |c| - family, qualifier = parse_column_name(c.to_s) - if qualifier - scan.addColumn(family, qualifier) - else - scan.addFamily(family) - end - end - - unless filter.class == String - scan.setFilter(filter) - else - scan.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - scan.setTimeStamp(timestamp) if timestamp - scan.setCacheBlocks(cache_blocks) - scan.setCaching(cache) if cache > 0 - scan.setMaxVersions(versions) if versions > 1 - scan.setTimeRange(timerange[0], timerange[1]) if timerange - scan.setRaw(raw) - else - scan = org.apache.hadoop.hbase.client.Scan.new - end - - # Start the scanner - scanner = @table.getScanner(scan) - count = 0 - res = {} - iter = scanner.iterator - - # Iterate results - while iter.hasNext - if limit > 0 && count >= limit - break - end - - row = iter.next - key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) - - row.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - cell = to_string(column, kv, maxlength) - - if block_given? - yield(key, "column=#{column}, #{cell}") - else - res[key] ||= {} - res[key][column] = cell - end - end - - # One more row processed - count += 1 - end - - return ((block_given?) ? count : res) - end - - #---------------------------------------------------------------------------------------- - # Helper methods - - # Returns a list of column names in the table - def get_all_columns - @table.table_descriptor.getFamilies.map do |family| - "#{family.getNameAsString}:" - end - end - - # Checks if current table is one of the 'meta' tables - def is_meta_table? - tn = @table.table_name - org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) || org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::ROOT_TABLE_NAME) - end - - # Returns family and (when has it) qualifier for a column name - def parse_column_name(column) - split = org.apache.hadoop.hbase.KeyValue.parseColumn(column.to_java_bytes) - return split[0], (split.length > 1) ? split[1] : nil - end - - # Make a String of the passed kv - # Intercept cells whose format we know such as the info:regioninfo in .META. - def to_string(column, kv, maxlength = -1) - if is_meta_table? - if column == 'info:regioninfo' or column == 'info:splitA' or column == 'info:splitB' - hri = org.apache.hadoop.hbase.util.Writables.getHRegionInfoOrNull(kv.getValue) - return "timestamp=%d, value=%s" % [kv.getTimestamp, hri.toString] - end - if column == 'info:serverstartcode' - if kv.getValue.length > 0 - str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValue) - else - str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValue) - end - return "timestamp=%d, value=%s" % [kv.getTimestamp, str_val] - end - end - - if kv.isDelete - val = "timestamp=#{kv.getTimestamp}, type=#{org.apache.hadoop.hbase.KeyValue::Type::codeToType(kv.getType)}" - else - val = "timestamp=#{kv.getTimestamp}, value=#{org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getValue)}" - end - (maxlength != -1) ? val[0, maxlength] : val - end - - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb deleted file mode 100644 index 584f70087f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -require 'rbconfig' - -module IRB - WINDOZE = Config::CONFIG['host_os'] =~ /mswin|mingw/ - - # Subclass of IRB so can intercept methods - class HIRB < Irb - def initialize - # This is ugly. Our 'help' method above provokes the following message - # on irb construction: 'irb: warn: can't alias help from irb_help.' - # Below, we reset the output so its pointed at /dev/null during irb - # construction just so this message does not come out after we emit - # the banner. Other attempts at playing with the hash of methods - # down in IRB didn't seem to work. I think the worst thing that can - # happen is the shell exiting because of failed IRB construction with - # no error (though we're not blanking STDERR) - begin - # Map the '/dev/null' according to the runing platform - # Under Windows platform the 'dev/null' is not fully compliant with unix, - # and the 'NUL' object need to be use instead. - devnull = "/dev/null" - devnull = "NUL" if WINDOZE - f = File.open(devnull, "w") - $stdout = f - super - ensure - f.close() - $stdout = STDOUT - end - end - - def output_value - # Suppress output if last_value is 'nil' - # Otherwise, when user types help, get ugly 'nil' - # after all output. - if @context.last_value != nil - super - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb deleted file mode 100644 index 1330b7e834..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb +++ /dev/null @@ -1,315 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Shell commands module -module Shell - @@commands = {} - def self.commands - @@commands - end - - @@command_groups = {} - def self.command_groups - @@command_groups - end - - def self.load_command(name, group) - return if commands[name] - - # Register command in the group - raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] - command_groups[group][:commands] << name - - # Load command - begin - require "shell/commands/#{name}" - klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize - commands[name] = eval("Commands::#{klass_name}") - rescue => e - raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" - end - end - - def self.load_command_group(group, opts) - raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] - - command_groups[group] = { - :commands => [], - :command_names => opts[:commands], - :full_name => opts[:full_name] || group, - :comment => opts[:comment] - } - - opts[:commands].each do |command| - load_command(command, group) - end - end - - #---------------------------------------------------------------------- - class Shell - attr_accessor :hbase - attr_accessor :formatter - - @debug = false - attr_accessor :debug - - def initialize(hbase, formatter) - self.hbase = hbase - self.formatter = formatter - end - - def hbase_admin - @hbase_admin ||= hbase.admin(formatter) - end - - def hbase_table(name) - hbase.table(name, formatter) - end - - def hbase_replication_admin - @hbase_replication_admin ||= hbase.replication_admin(formatter) - end - - def hbase_security_admin - @hbase_security_admin ||= hbase.security_admin(formatter) - end - - def export_commands(where) - ::Shell.commands.keys.each do |cmd| - where.send :instance_eval, <<-EOF - def #{cmd}(*args) - @shell.command('#{cmd}', *args) - puts - end - EOF - end - end - - def command_instance(command) - ::Shell.commands[command.to_s].new(self) - end - - def command(command, *args) - command_instance(command).command_safe(self.debug, *args) - end - - def print_banner - puts "HBase Shell; enter 'help' for list of supported commands." - puts 'Type "exit" to leave the HBase Shell' - print 'Version ' - command('version') - puts - end - - def help_multi_command(command) - puts "Command: #{command}" - puts command_instance(command).help - puts - return nil - end - - def help_command(command) - puts command_instance(command).help - return nil - end - - def help_group(group_name) - group = ::Shell.command_groups[group_name.to_s] - group[:commands].sort.each { |cmd| help_multi_command(cmd) } - if group[:comment] - puts '-' * 80 - puts - puts group[:comment] - puts - end - return nil - end - - def help(command = nil) - if command - return help_command(command) if ::Shell.commands[command.to_s] - return help_group(command) if ::Shell.command_groups[command.to_s] - puts "ERROR: Invalid command or command group name: #{command}" - puts - end - - puts help_header - puts - puts 'COMMAND GROUPS:' - ::Shell.command_groups.each do |name, group| - puts " Group name: " + name - puts " Commands: " + group[:command_names].sort.join(', ') - puts - end - unless command - puts 'SHELL USAGE:' - help_footer - end - return nil - end - - def help_header - return "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + "\n" + - "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" + - "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." - end - - def help_footer - puts <<-HERE -Quote all names in HBase Shell such as table and column names. Commas delimit -command parameters. Type after entering a command to run it. -Dictionaries of configuration used in the creation and alteration of tables are -Ruby Hashes. They look like this: - - {'key1' => 'value1', 'key2' => 'value2', ...} - -and are opened and closed with curley-braces. Key/values are delimited by the -'=>' character combination. Usually keys are predefined constants such as -NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type -'Object.constants' to see a (messy) list of all constants in the environment. - -If you are using binary keys or values and need to enter them in the shell, use -double-quote'd hexadecimal representation. For example: - - hbase> get 't1', "key\\x03\\x3f\\xcd" - hbase> get 't1', "key\\003\\023\\011" - hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" - -The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. -For more on the HBase Shell, see http://hbase.apache.org/docs/current/book.html - HERE - end - end -end - -# Load commands base class -require 'shell/commands' - -# Load all commands -Shell.load_command_group( - 'general', - :full_name => 'GENERAL HBASE SHELL COMMANDS', - :commands => %w[ - status - version - whoami - ] -) - -Shell.load_command_group( - 'ddl', - :full_name => 'TABLES MANAGEMENT COMMANDS', - :commands => %w[ - alter - create - describe - disable - disable_all - is_disabled - drop - drop_all - enable - enable_all - is_enabled - exists - list - show_filters - alter_status - alter_async - ] -) - -Shell.load_command_group( - 'dml', - :full_name => 'DATA MANIPULATION COMMANDS', - :commands => %w[ - count - delete - deleteall - get - get_counter - incr - put - scan - truncate - ] -) - -Shell.load_command_group( - 'tools', - :full_name => 'HBASE SURGERY TOOLS', - :comment => "WARNING: Above commands are for 'experts'-only as misuse can damage an install", - :commands => %w[ - assign - balancer - balance_switch - close_region - compact - flush - major_compact - move - split - unassign - zk_dump - hlog_roll - ] -) - -Shell.load_command_group( - 'replication', - :full_name => 'CLUSTER REPLICATION TOOLS', - :comment => "In order to use these tools, hbase.replication must be true. enabling/disabling is currently unsupported", - :commands => %w[ - add_peer - remove_peer - list_peers - enable_peer - disable_peer - start_replication - stop_replication - list_replicated_tables - ] -) - -Shell.load_command_group( - 'snapshot', - :full_name => 'CLUSTER SNAPSHOT TOOLS', - :commands => %w[ - snapshot - clone_snapshot - restore_snapshot - delete_snapshot - list_snapshots - ] -) - -Shell.load_command_group( - 'security', - :full_name => 'SECURITY TOOLS', - :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", - :commands => %w[ - grant - revoke - user_permission - ] -) - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb deleted file mode 100644 index af6df33440..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2009 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Command - attr_accessor :shell - - def initialize(shell) - self.shell = shell - end - - def command_safe(debug, *args) - translate_hbase_exceptions(*args) { command(*args) } - rescue => e - puts - puts "ERROR: #{e}" - puts "Backtrace: #{e.backtrace.join("\n ")}" if debug - puts - puts "Here is some help for this command:" - puts help - puts - ensure - return nil - end - - def admin - shell.hbase_admin - end - - def table(name) - shell.hbase_table(name) - end - - def replication_admin - shell.hbase_replication_admin - end - - def security_admin - shell.hbase_security_admin - end - - #---------------------------------------------------------------------- - - def formatter - shell.formatter - end - - def format_simple_command - now = Time.now - yield - formatter.header - formatter.footer(now) - end - - def translate_hbase_exceptions(*args) - yield - rescue org.apache.hadoop.hbase.TableNotFoundException - raise "Unknown table #{args.first}!" - rescue org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException - valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } - raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" - rescue org.apache.hadoop.hbase.TableExistsException - raise "Table already exists: #{args.first}!" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb deleted file mode 100644 index 3be8f0a427..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Alter < Command - def help - return <<-EOF -Alter column family schema; pass table name and a dictionary -specifying new column family schema. Dictionaries are described -on the main help command output. Dictionary must include name -of column family to alter. For example, - -To change or add the 'f1' column family in table 't1' from defaults -to instead keep a maximum of 5 cell VERSIONS, do: - - hbase> alter 't1', NAME => 'f1', VERSIONS => 5 - -To delete the 'f1' column family in table 't1', do: - - hbase> alter 't1', NAME => 'f1', METHOD => 'delete' - -or a shorter version: - - hbase> alter 't1', 'delete' => 'f1' - -You can also change the column family config by set attribute CONFIG like this: - hbase> alter 'test', NAME=>'f', CONFIG => {'hbase.hstore.compaction.min' => '5'} - -You can also change table-scope attributes like MAX_FILESIZE -MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. - -For example, to change the max size of a family to 128MB, do: - - hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' - -You can also change the table-scope by set attribute CONFIG like this: - hbase> alter 'test', METHOD=>'table_att', CONFIG => {'hbase.hstore.compaction.min' => '5'} - -You can add a table coprocessor by setting a table coprocessor attribute: - - hbase> alter 't1', METHOD => 'table_att', - 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' - -Since you can have multiple coprocessors configured for a table, a -sequence number will be automatically appended to the attribute name -to uniquely identify it. - -The coprocessor attribute must match the pattern below in order for -the framework to understand how to load the coprocessor classes: - - [coprocessor jar file location] | class name | [priority] | [arguments] - -You can also remove a table-scope attribute: - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' - -There could be more than one alteration in one command: - - hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, true, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb deleted file mode 100644 index 01dfd99465..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterAsync < Command - def help - return <<-EOF -Alter column family schema, does not wait for all regions to receive the -schema changes. Pass table name and a dictionary specifying new column -family schema. Dictionaries are described on the main help command output. -Dictionary must include name of column family to alter. For example, - -To change or add the 'f1' column family in table 't1' from defaults -to instead keep a maximum of 5 cell VERSIONS, do: - - hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 - -To delete the 'f1' column family in table 't1', do: - - hbase> alter_async 't1', NAME => 'f1', METHOD => 'delete' - -or a shorter version: - - hbase> alter_async 't1', 'delete' => 'f1' - -You can also change table-scope attributes like MAX_FILESIZE -MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. - -For example, to change the max size of a family to 128MB, do: - - hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' - -There could be more than one alteration in one command: - - hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} - -To check if all the regions have been updated, use alter_status -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, false, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb deleted file mode 100644 index cc79e1108b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterStatus < Command - def help - return <<-EOF -Get the status of the alter command. Indicates the number of regions of the -table that have received the updated schema -Pass table name. - -hbase> alter_status 't1' -EOF - end - def command(table) - admin.alter_status(table) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb deleted file mode 100644 index 43d240f12f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloneSnapshot < Command - def help - return <<-EOF -Create a new table by cloning the snapshot content. -There're no copies of data involved. -And writing on the newly created table will not influence the snapshot data. - -Examples: - hbase> clone_snapshot 'snapshotName', 'tableName' -EOF - end - - def command(snapshot_name, table) - format_simple_command do - admin.clone_snapshot(snapshot_name, table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb deleted file mode 100644 index f1010ac6ca..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloseRegion < Command - def help - return <<-EOF -Close a single region. Ask the master to close a region out on the cluster -or if 'SERVER_NAME' is supplied, ask the designated hosting regionserver to -close the region directly. Closing a region, the master expects 'REGIONNAME' -to be a fully qualified region name. When asking the hosting regionserver to -directly close a region, you pass the regions' encoded name only. A region -name looks like this: - - TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. - -The trailing period is part of the regionserver name. A region's encoded name -is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 -(without the period). A 'SERVER_NAME' is its host, port plus startcode. For -example: host187.example.com,60020,1289493121758 (find servername in master ui -or when you do detailed status in shell). This command will end up running -close on the region hosting regionserver. The close is done without the -master's involvement (It will not know of the close). Once closed, region will -stay closed. Use assign to reopen/reassign. Use unassign or move to assign -the region elsewhere on cluster. Use with caution. For experts only. -Examples: - - hbase> close_region 'REGIONNAME' - hbase> close_region 'REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(region_name, server = nil) - format_simple_command do - admin.close_region(region_name, server) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb deleted file mode 100644 index fa0b82eef8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Compact < Command - def help - return <<-EOF - Compact all regions in passed table or pass a region row - to compact an individual region. You can also compact a single column - family within a region. - Examples: - Compact all regions in a table: - hbase> compact 't1' - Compact an entire region: - hbase> compact 'r1' - Compact only a column family within a region: - hbase> compact 'r1', 'c1' - Compact a column family within a table: - hbase> compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb deleted file mode 100644 index 6596441c1c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Count < Command - def help - return <<-EOF -Count the number of rows in a table. This operation may take a LONG -time (Run '$HADOOP_HOME/bin/hadoop jar hbase.jar rowcount' to run a -counting mapreduce job). Current count is shown every 1000 rows by -default. Count interval may be optionally specified. Scan caching -is enabled on count scans by default. Default cache size is 10 rows. -If your rows are small in size, you may want to increase this -parameter. Examples: - - hbase> count 't1' - hbase> count 't1', INTERVAL => 100000 - hbase> count 't1', CACHE => 1000 - hbase> count 't1', INTERVAL => 10, CACHE => 1000 -EOF - end - - def command(table, params = {}) - # If the second parameter is an integer, then it is the old command syntax - params = { 'INTERVAL' => params } if params.kind_of?(Fixnum) - - # Merge params with defaults - params = { - 'INTERVAL' => 1000, - 'CACHE' => 10 - }.merge(params) - - # Call the counter method - now = Time.now - formatter.header - count = table(table).count(params['INTERVAL'].to_i, params['CACHE'].to_i) do |cnt, row| - formatter.row([ "Current count: #{cnt}, row: #{row}" ]) - end - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb deleted file mode 100644 index 14c1b0ff4f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Create < Command - def help - return <<-EOF -Create table; pass table name, a dictionary of specifications per -column family, and optionally a dictionary of table configuration. -Dictionaries are described below in the GENERAL NOTES section. -Examples: - - hbase> create 't1', {NAME => 'f1', VERSIONS => 5} - hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} - hbase> # The above in shorthand would be the following: - hbase> create 't1', 'f1', 'f2', 'f3' - hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} - hbase> create 't1', 'f1', {SPLITS => ['10', '20', '30', '40']} - hbase> create 't1', 'f1', {SPLITS_FILE => 'splits.txt'} - hbase> # Optionally pre-split the table into NUMREGIONS, using - hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) - hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} -EOF - end - - def command(table, *args) - format_simple_command do - admin.create(table, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb deleted file mode 100644 index 12bc405933..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Delete < Command - def help - return <<-EOF -Put a delete cell value at specified table/row/column and optionally -timestamp coordinates. Deletes must match the deleted cell's -coordinates exactly. When scanning, a delete cell suppresses older -versions. To delete a cell from 't1' at row 'r1' under column 'c1' -marked with the time 'ts1', do: - - hbase> delete 't1', 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table(table).delete(row, column, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb deleted file mode 100644 index 5731b606b4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Deleteall < Command - def help - return <<-EOF -Delete all cells in a given row; pass a table name, row, and optionally -a column and timestamp. Examples: - - hbase> deleteall 't1', 'r1' - hbase> deleteall 't1', 'r1', 'c1' - hbase> deleteall 't1', 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table(table).deleteall(row, column, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb deleted file mode 100644 index 34c5f9ca7d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Disable < Command - def help - return <<-EOF -Start disable of named table: e.g. "hbase> disable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.disable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb deleted file mode 100644 index b7c9cf5fdd..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DisableAll < Command - def help - return <<-EOF -Disable all of tables matching the given regex: - -hbase> disable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.disable_all(regex) - puts "#{count - failed.size} tables successfully disabled" - puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb deleted file mode 100644 index dcbefa3234..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DropAll < Command - def help - return <<-EOF -Drop all of the tables matching the given regex: - -hbase> drop_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.drop_all(regex) - puts "#{count - failed.size} tables successfully dropped" - puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb deleted file mode 100644 index a0dc340f5d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Enable < Command - def help - return <<-EOF -Start enable of named table: e.g. "hbase> enable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.enable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb deleted file mode 100644 index ee016373f4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class EnableAll < Command - def help - return <<-EOF -Enable all of the tables matching the given regex: - -hbase> enable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.enable_all(regex) - puts "#{count - failed.size} tables successfully enabled" - puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb deleted file mode 100644 index f35f197238..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Exists < Command - def help - return <<-EOF -Does the named table exist? e.g. "hbase> exists 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - "Table #{table} " + (admin.exists?(table.to_s) ? "does exist" : "does not exist") - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb deleted file mode 100644 index 8456d58a5c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Get < Command - def help - return <<-EOF -Get row or cell contents; pass table name, row, and optionally -a dictionary of column(s), timestamp, timerange and versions. Examples: - - hbase> get 't1', 'r1' - hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} - hbase> get 't1', 'r1', {COLUMN => 'c1'} - hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} - hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} - hbase> get 't1', 'r1', 'c1' - hbase> get 't1', 'r1', 'c1', 'c2' - hbase> get 't1', 'r1', ['c1', 'c2'] -EOF - end - - def command(table, row, *args) - now = Time.now - formatter.header(["COLUMN", "CELL"]) - - table(table).get(row, *args) do |column, value| - formatter.row([ column, value ]) - end - - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb deleted file mode 100644 index 3cbe2268d9..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class GetCounter < Command - def help - return <<-EOF -Return a counter cell value at specified table/row/column coordinates. -A cell cell should be managed with atomic increment function oh HBase -and the data should be binary encoded. Example: - - hbase> get_counter 't1', 'r1', 'c1' -EOF - end - - def command(table, row, column, value = nil) - if cnt = table(table).get_counter(row, column) - puts "COUNTER VALUE = #{cnt}" - else - puts "No counter found at specified coordinates" - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb deleted file mode 100644 index 38a2fc5179..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Incr < Command - def help - return <<-EOF -Increments a cell 'value' at specified table/row/column coordinates. -To increment a cell value in table 't1' at row 'r1' under column -'c1' by 1 (can be omitted) or 10 do: - - hbase> incr 't1', 'r1', 'c1' - hbase> incr 't1', 'r1', 'c1', 1 - hbase> incr 't1', 'r1', 'c1', 10 -EOF - end - - def command(table, row, column, value = nil) - cnt = table(table).incr(row, column, value) - puts "COUNTER VALUE = #{cnt}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb deleted file mode 100644 index 9d3c7ee747..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsDisabled < Command - def help - return <<-EOF -Is named table disabled?: e.g. "hbase> is_disabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.disabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb deleted file mode 100644 index 96b2b152c2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsEnabled < Command - def help - return <<-EOF -Is named table enabled?: e.g. "hbase> is_enabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.enabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb deleted file mode 100644 index 592fb5eec2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class List < Command - def help - return <<-EOF -List all tables in hbase. Optional regular expression parameter could -be used to filter the output. Examples: - - hbase> list - hbase> list 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "TABLE" ]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - list.each do |table| - formatter.row([ table ]) - end - - formatter.footer(now, list.size) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb deleted file mode 100644 index b1494b8d3a..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListReplicatedTables< Command - def help - return <<-EOF -List all the tables and column families replicated from this cluster - - hbase> list_replicated_tables - hbase> list_replicated_tables 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - - formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ]) - list = replication_admin.list_replicated_tables - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))} - list.each do |e| - if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL - replicateType = "GLOBAL" - else - replicateType = "unknown" - end - formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32]) - end - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb deleted file mode 100644 index 325344b526..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class MajorCompact < Command - def help - return <<-EOF - Run major compaction on passed table or pass a region row - to major compact an individual region. To compact a single - column family within a region specify the region name - followed by the column family name. - Examples: - Compact all regions in a table: - hbase> major_compact 't1' - Compact an entire region: - hbase> major_compact 'r1' - Compact a single column family within a region: - hbase> major_compact 'r1', 'c1' - Compact a single column family within a table: - hbase> major_compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.major_compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb deleted file mode 100644 index 0e3db8fe47..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Move < Command - def help - return <<-EOF -Move a region. Optionally specify target regionserver else we choose one -at random. NOTE: You pass the encoded region name, not the region name so -this command is a little different to the others. The encoded region name -is the hash suffix on region names: e.g. if the region name were -TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then -the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 -A server name is its host, port plus startcode. For example: -host187.example.com,60020,1289493121758 -Examples: - - hbase> move 'ENCODED_REGIONNAME' - hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(encoded_region_name, server_name = nil) - format_simple_command do - admin.move(encoded_region_name, server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb deleted file mode 100644 index dde0433708..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Put < Command - def help - return <<-EOF -Put a cell 'value' at specified table/row/column and optionally -timestamp coordinates. To put a cell value into table 't1' at -row 'r1' under column 'c1' marked with the time 'ts1', do: - - hbase> put 't1', 'r1', 'c1', 'value', ts1 -EOF - end - - def command(table, row, column, value, timestamp = nil) - format_simple_command do - table(table).put(row, column, value, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb deleted file mode 100644 index 518e628d67..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Scan < Command - def help - return <<-EOF -Scan a table; pass table name and optionally a dictionary of scanner -specifications. Scanner specifications may include one or more of: -TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, -or COLUMNS, CACHE - -If no columns are specified, all columns will be scanned. -To scan all members of a column family, leave the qualifier empty as in -'col_family:'. - -The filter can be specified in two ways: -1. Using a filterString - more information on this is available in the -Filter Language document attached to the HBASE-4176 JIRA -2. Using the entire package name of the filter. - -Some examples: - - hbase> scan '.META.' - hbase> scan '.META.', {COLUMNS => 'info:regioninfo'} - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} - hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} - hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} - hbase> scan 't1', {FILTER => org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} - -For experts, there is an additional option -- CACHE_BLOCKS -- which -switches block caching for the scanner on (true) or off (false). By -default it is enabled. Examples: - - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} - -Also for experts, there is an advanced option -- RAW -- which instructs the -scanner to return all cells (including delete markers and uncollected deleted -cells). This option cannot be combined with requesting specific COLUMNS. -Disabled by default. Example: - - hbase> scan 't1', {RAW => true, VERSIONS => 10} -EOF - end - - def command(table, args = {}) - now = Time.now - formatter.header(["ROW", "COLUMN+CELL"]) - - count = table(table).scan(args) do |row, cells| - formatter.row([ row, cells ]) - end - - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb deleted file mode 100644 index 1581db5b37..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -java_import org.apache.hadoop.hbase.filter.ParseFilter - -module Shell - module Commands - class ShowFilters < Command - def help - return <<-EOF -Show all the filters in hbase. Example: - hbase> show_filters - - Documentation on filters mentioned below can be found at: https://our.intern.facebook.com/intern/wiki/index.php/HBase/Filter_Language - ColumnPrefixFilter - TimestampsFilter - PageFilter - ..... - KeyOnlyFilter -EOF - end - - def command( ) - now = Time.now - formatter.row(["Documentation on filters mentioned below can " + - "be found at: https://our.intern.facebook.com/intern/" + - "wiki/index.php/HBase/Filter_Language"]) - - parseFilter = ParseFilter.new - supportedFilters = parseFilter.getSupportedFilters - - supportedFilters.each do |filter| - formatter.row([filter]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb deleted file mode 100644 index 1c4ecfec0e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Snapshot < Command - def help - return <<-EOF -Take a snapshot of specified table. Examples: - - hbase> snapshot 'sourceTable', 'snapshotName' -EOF - end - - def command(table, snapshot_name) - format_simple_command do - admin.snapshot(table, snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb deleted file mode 100644 index 47e21f86c2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Split < Command - def help - return <<-EOF -Split entire table or pass a region to split individual region. With the -second parameter, you can specify an explicit split key for the region. -Examples: - split 'tableName' - split 'regionName' # format: 'tableName,startKey,id' - split 'tableName', 'splitKey' - split 'regionName', 'splitKey' -EOF - end - - def command(table_or_region_name, split_point = nil) - format_simple_command do - admin.split(table_or_region_name, split_point) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb deleted file mode 100644 index 5d1cd1b5ee..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class StartReplication < Command - def help - return <<-EOF -Restarts all the replication features. The state in which each -stream starts in is undetermined. -WARNING: -start/stop replication is only meant to be used in critical load situations. -Examples: - - hbase> start_replication -EOF - end - - def command - format_simple_command do - replication_admin.start_replication - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb deleted file mode 100644 index f5074d73a8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class StopReplication < Command - def help - return <<-EOF -Stops all the replication features. The state in which each -stream stops in is undetermined. -WARNING: -start/stop replication is only meant to be used in critical load situations. -Examples: - - hbase> stop_replication -EOF - end - - def command - format_simple_command do - replication_admin.stop_replication - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb deleted file mode 100644 index a24e167b98..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Truncate < Command - def help - return <<-EOF - Disables, drops and recreates the specified table. -EOF - end - - def command(table) - format_simple_command do - puts "Truncating '#{table}' table (it may take a while):" - admin.truncate(table) { |log| puts " - #{log}" } - end - end - - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb deleted file mode 100644 index 75242baf4b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Unassign < Command - def help - return <<-EOF -Unassign a region. Unassign will close region in current location and then -reopen it again. Pass 'true' to force the unassignment ('force' will clear -all in-memory state in master before the reassign. If results in -double assignment use hbck -fix to resolve. To be used by experts). -Use with caution. For expert use only. Examples: - - hbase> unassign 'REGIONNAME' - hbase> unassign 'REGIONNAME', true -EOF - end - - def command(region_name, force = 'false') - format_simple_command do - admin.unassign(region_name, force) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb deleted file mode 100644 index 372b0dc341..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Version < Command - def help - return <<-EOF -Output this HBase version -EOF - end - - def command - # Output version. - puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb deleted file mode 100644 index bb23962df8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ZkDump < Command - def help - return <<-EOF -Dump status of HBase cluster as seen by ZooKeeper. -EOF - end - - def command - puts admin.zk_dump - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb deleted file mode 100644 index ea17882a6e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb +++ /dev/null @@ -1,171 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Results formatter -module Shell - module Formatter - # Base abstract class for results formatting. - class Base - attr_reader :row_count - - def is_valid_io?(obj) - obj.instance_of?(IO) || obj == Kernel - end - - def refresh_width() - if $stdout.tty? - @max_width = Java::jline.Terminal.getTerminal().getTerminalWidth() - else - @max_width = 0 - end - end - - # Takes an output stream and a print width. - def initialize(opts = {}) - options = { - :output_stream => Kernel, - }.merge(opts) - - @out = options[:output_stream] - refresh_width - @row_count = 0 - - # raise an error if the stream is not valid - raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) - end - - def header(args = [], widths = []) - refresh_width - row(args, false, widths) if args.length > 0 - @row_count = 0 - end - - # Output a row. - # Inset is whether or not to offset row by a space. - def row(args = [], inset = true, widths = []) - # Print out nothing - return if !args || args.empty? - - # Print a string - if args.is_a?(String) - output(args) - @out.puts - return - end - - # TODO: Look at the type. Is it RowResult? - if args.length == 1 - splits = split(@max_width, dump(args[0])) - for l in splits - output(@max_width, l) - @out.puts - end - elsif args.length == 2 - if @max_width == 0 - col1width = col2width = 0 - else - col1width = (not widths or widths.length == 0) ? @max_width / 4 : @max_width * widths[0] / 100 - col2width = (not widths or widths.length < 2) ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 - end - splits1 = split(col1width, dump(args[0])) - splits2 = split(col2width, dump(args[1])) - biggest = (splits2.length > splits1.length)? splits2.length: splits1.length - index = 0 - while index < biggest - # Inset by one space if inset is set. - @out.print(" ") if inset - output(col1width, splits1[index]) - # Add extra space so second column lines up w/ second column output - @out.print(" ") unless inset - @out.print(" ") - output(col2width, splits2[index]) - index += 1 - @out.puts - end - else - # Print a space to set off multi-column rows - print ' ' - first = true - for e in args - @out.print " " unless first - first = false - @out.print e - end - puts - end - @row_count += 1 - end - - def split(width, str) - if width == 0 - return [str] - end - result = [] - index = 0 - while index < str.length do - result << str.slice(index, width) - index += width - end - result - end - - def dump(str) - return if str.instance_of?(Fixnum) - # Remove double-quotes added by 'dump'. - return str - end - - def output(str) - output(@max_width, str) - end - - def output(width, str) - if str == nil - str = '' - end - if not width or width == str.length - @out.print(str) - else - @out.printf('%-*s', width, str) - end - end - - def footer(start_time = nil, row_count = nil) - return unless start_time - row_count ||= @row_count - # Only output elapsed time and row count if startTime passed - @out.puts("%d row(s) in %.4f seconds" % [row_count, Time.now - start_time]) - end - end - - - class Console < Base - end - - class XHTMLFormatter < Base - # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html - # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html - end - - class JSON < Base - end - end -end - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh deleted file mode 100755 index 6f3cd9085d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -# Start hadoop hbase daemons. -# Run this on master node. -usage="Usage: start-hbase.sh" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` - - -if [ "$distMode" == 'false' ] -then - "$bin"/hbase-daemon.sh start master -else - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" start regionserver - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh deleted file mode 100755 index e9749fee28..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -# Stop hadoop hbase daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# variables needed for stop command -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid - -echo -n stopping hbase -echo "`date` Stopping hbase (via master)" >> $loglog - -nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - master stop "$@" > "$logout" 2>&1 < /dev/null & - -while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; -done -# Add a CR after we're done w/ dots. -echo - -# distributed == false means that the HMaster will kill ZK when it exits -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'true' ] -then - # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message - # stop any backup masters - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch b/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch deleted file mode 100644 index 94906ceab4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch +++ /dev/null @@ -1,28 +0,0 @@ ---- hbase.orig 2013-10-23 18:14:51.867728302 -0400 -+++ hbase 2013-10-23 20:11:57.564530457 -0400 -@@ -137,7 +137,24 @@ - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then -- ${MVN} -f "${HBASE_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ # ------------------- -+ # Titan build support -+ # ------------------- -+ # Check for Maven command line options file. -+ # If titan-hbase has already been built, then this file will contain options -+ # to target the version of Hadoop used during compilation. This is -+ # necessary when the user compiles titan-hbase with a non-default Hadoop -+ # version (e.g. 2.2.0). If we omitted this check, then we'd try to start -+ # HBase with the default Hadoop version (currentl 1.0.4), and it would break. -+ MVN_OPTS= -+ optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" -+ unset IFS -+ if [ -f "${optf}" ] -+ then -+ MVN_OPTS=`cat "${optf}"` -+ fi -+ ${MVN} -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` - } diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch b/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch deleted file mode 100644 index 0d4dc74cae..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/titan-hbase/bin/hirb.rb b/titan-hbase/bin/hirb.rb -index 32a51b3..c792515 100755 ---- a/titan-hbase/bin/hirb.rb -+++ b/titan-hbase/bin/hirb.rb -@@ -37,12 +37,9 @@ include Java - # Some goodies for hirb. Should these be left up to the user's discretion? - require 'irb/completion' - --# Add the $HBASE_HOME/lib/ruby OR $HBASE_HOME/src/main/ruby/lib directory --# to the ruby load path so I can load up my HBase ruby modules --if File.exists?(File.join(File.dirname(__FILE__), "..", "lib", "ruby", "hbase.rb")) -- $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib", "ruby") --else -- $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "src", "main", "ruby") -+# Add HBase to the ruby load path -+if File.exists?(File.join(File.dirname(__FILE__), "ruby", "hbase.rb")) -+ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "ruby") - end - - # diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java deleted file mode 100644 index 6c385d56f9..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; - -public class HBaseAdmin0_96 implements AdminMask -{ - - private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_96.class); - - private final HBaseAdmin adm; - - public HBaseAdmin0_96(HBaseAdmin adm) - { - this.adm = adm; - } - - @Override - public void clearTable(String tableName, long timestamp) throws IOException - { - if (!adm.tableExists(tableName)) { - log.debug("clearStorage() called before table {} was created, skipping.", tableName); - return; - } - -// long before = System.currentTimeMillis(); -// try { -// adm.disableTable(tableName); -// adm.deleteTable(tableName); -// } catch (IOException e) { -// throw new PermanentBackendException(e); -// } -// ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); -// long after = System.currentTimeMillis(); -// logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before); - - - // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than - // disabling and deleting tables. - HTable table = null; - - try { - table = new HTable(adm.getConfiguration(), tableName); - - Scan scan = new Scan(); - scan.setBatch(100); - scan.setCacheBlocks(false); - scan.setCaching(2000); - scan.setTimeRange(0, Long.MAX_VALUE); - scan.setMaxVersions(1); - - ResultScanner scanner = null; - - try { - scanner = table.getScanner(scan); - - for (Result res : scanner) { - Delete d = new Delete(res.getRow()); - - d.setTimestamp(timestamp); - table.delete(d); - } - } finally { - IOUtils.closeQuietly(scanner); - } - } finally { - IOUtils.closeQuietly(table); - } - } - - @Override - public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException - { - return adm.getTableDescriptor(tableName.getBytes()); - } - - @Override - public boolean tableExists(String tableName) throws IOException - { - return adm.tableExists(tableName); - } - - @Override - public void createTable(HTableDescriptor desc) throws IOException - { - adm.createTable(desc); - } - - @Override - public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException - { - adm.createTable(desc, startKey, endKey, numRegions); - } - - @Override - public int getEstimatedRegionServerCount() - { - int serverCount = -1; - try { - serverCount = adm.getClusterStatus().getServers().size(); - log.debug("Read {} servers from HBase ClusterStatus", serverCount); - } catch (IOException e) { - log.debug("Unable to retrieve HBase cluster status", e); - } - return serverCount; - } - - @Override - public void disableTable(String tableName) throws IOException - { - adm.disableTable(tableName); - } - - @Override - public void enableTable(String tableName) throws IOException - { - adm.enableTable(tableName); - } - - @Override - public boolean isTableDisabled(String tableName) throws IOException - { - return adm.isTableDisabled(tableName); - } - - @Override - public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException - { - adm.addColumn(tableName, columnDescriptor); - } - - @Override - public void close() throws IOException - { - adm.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java deleted file mode 100644 index 94b0e20a05..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; - -public class HConnection0_96 implements ConnectionMask -{ - - private final HConnection cnx; - - public HConnection0_96(HConnection cnx) - { - this.cnx = cnx; - } - - @Override - public TableMask getTable(String name) throws IOException - { - return new HTable0_96(cnx.getTable(name)); - } - - @Override - public AdminMask getAdmin() throws IOException - { - return new HBaseAdmin0_96(new HBaseAdmin(cnx)); - } - - @Override - public void close() throws IOException - { - cnx.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java deleted file mode 100644 index 4845c5c9eb..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Row; -import org.apache.hadoop.hbase.client.Scan; - -public class HTable0_96 implements TableMask -{ - private final HTableInterface table; - - public HTable0_96(HTableInterface table) - { - this.table = table; - } - - @Override - public ResultScanner getScanner(Scan filter) throws IOException - { - return table.getScanner(filter); - } - - @Override - public Result[] get(List gets) throws IOException - { - return table.get(gets); - } - - @Override - public void batch(List writes, Object[] results) throws IOException, InterruptedException - { - table.batch(writes, results); - table.flushCommits(); - } - - @Override - public void close() throws IOException - { - table.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh deleted file mode 100755 index 843eaaa74f..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [--hosts regionserversfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. $bin/hbase-config.sh - -remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" -args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" - -command=$2 -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (master-backup) - exec "$bin/master-backup.sh" $args - ;; - (*) - exec "$bin/regionservers.sh" $args - ;; -esac - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby deleted file mode 100755 index 37bce46f6b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - - -`dirname $0`/hbase org.jruby.Main $* - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh deleted file mode 100755 index 114757c763..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2010 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all backup master hosts. -# -# Environment Variables -# -# HBASE_BACKUP_MASTERS File naming remote hosts. -# Default is ${HBASE_CONF_DIR}/backup-masters -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the master backup file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_BACKUP_MASTERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_BACKUP_MASTERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" - else - export HOSTLIST="${HBASE_BACKUP_MASTERS}" - fi -fi - - -args=${@// /\\ } -args=${args/master-backup/master} - -if [ -f $HOSTLIST ]; then - for hmaster in `cat "$HOSTLIST"`; do - ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ - 2>&1 | sed "s/^/$hmaster: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh deleted file mode 100755 index 9759f2b00c..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: regionservers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the regionservers file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_REGIONSERVERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_REGIONSERVERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/regionservers" - else - export HOSTLIST="${HBASE_REGIONSERVERS}" - fi -fi - -for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi -done - -wait diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb deleted file mode 100644 index baa6aeaeda..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class RepAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @replication_admin = org.apache.hadoop.hbase.client.replication.ReplicationAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Add a new peer cluster to replicate to - def add_peer(id, cluster_key) - @replication_admin.addPeer(id, cluster_key) - end - - #---------------------------------------------------------------------------------------------- - # Remove a peer cluster, stops the replication - def remove_peer(id) - @replication_admin.removePeer(id) - end - - - #--------------------------------------------------------------------------------------------- - # Show replcated tables/column families, and their ReplicationType - def list_replicated_tables - @replication_admin.listReplicated() - end - - #---------------------------------------------------------------------------------------------- - # List all peer clusters - def list_peers - @replication_admin.listPeers - end - - #---------------------------------------------------------------------------------------------- - # Get peer cluster state - def get_peer_state(id) - @replication_admin.getPeerState(id) ? "ENABLED" : "DISABLED" - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication stream to the specified peer - def enable_peer(id) - @replication_admin.enablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Stop the replication stream to the specified peer - def disable_peer(id) - @replication_admin.disablePeer(id) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb deleted file mode 100644 index deb5c5def5..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AddPeer< Command - def help - return <<-EOF -Add a peer cluster to replicate to, the id must be a short and -the cluster key is composed like this: -hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent -This gives a full path for HBase to connect to another cluster. -Examples: - - hbase> add_peer '1', "server1.cie.com:2181:/hbase" - hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" -EOF - end - - def command(id, cluster_key) - format_simple_command do - replication_admin.add_peer(id, cluster_key) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb deleted file mode 100644 index cce82c4770..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Describe < Command - def help - return <<-EOF -Describe the named table. For example: - hbase> describe 't1' -EOF - end - - def command(table) - now = Time.now - - desc = admin.describe(table) - - formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) - formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb deleted file mode 100644 index 7bdc95943b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Grant < Command - def help - return <<-EOF -Grant users specific rights. -Syntax : grant [ [ []] - -permissions is either zero or more letters from the set "RWXCA". -READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') - -For example: - - hbase> grant 'bobsmith', 'RWXCA' - hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' -EOF - end - - def command(user, rights, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.grant(user, rights, table_name, family, qualifier) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb deleted file mode 100644 index b292791541..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -module Shell - module Commands - class HlogRoll < Command - def help - return <<-EOF -Roll the log writer. That is, start writing log messages to a new file. -The name of the regionserver should be given as the parameter. A -'server_name' is the host, port plus startcode of a regionserver. For -example: host187.example.com,60020,1289493121758 (find servername in -master ui or when you do detailed status in shell) -EOF - end - - def command(server_name) - format_simple_command do - admin.hlog_roll(server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb deleted file mode 100644 index 95135185c2..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'time' - -module Shell - module Commands - class ListSnapshots < Command - def help - return <<-EOF -List all snapshots taken (by printing the names and relative information). -Optional regular expression parameter could be used to filter the output -by snapshot name. - -Examples: - hbase> list_snapshots - hbase> list_snapshots 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_snapshot.select {|s| regex.match(s.getName)} - list.each do |snapshot| - creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s - formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) - end - - formatter.footer(now, list.size) - return list.map { |s| s.getName() } - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb deleted file mode 100644 index ac9e23cbea..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RenameSnapshot < Command - def help - return <<-EOF -Rename a specified snapshot. Examples: - - hbase> rename_snapshot 'oldSnapshotName' 'newSnapshotName' -EOF - end - - def command(old_snapshot_name, new_snapshot_name) - format_simple_command do - admin.rename_snapshot(old_snapshot_name, new_snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb deleted file mode 100644 index 4d531711bc..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RestoreSnapshot < Command - def help - return <<-EOF -Restore a specified snapshot. -The restore will replace the content of the original table, -bringing back the content to the snapshot state. -The table must be disabled. - -Examples: - hbase> restore_snapshot 'snapshotName' -EOF - end - - def command(snapshot_name) - format_simple_command do - admin.restore_snapshot(snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb deleted file mode 100644 index f72c13caef..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Status < Command - def help - return <<-EOF -Show cluster status. Can be 'summary', 'simple', or 'detailed'. The -default is 'summary'. Examples: - - hbase> status - hbase> status 'simple' - hbase> status 'summary' - hbase> status 'detailed' -EOF - end - - def command(format = 'summary') - admin.status(format) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb deleted file mode 100644 index ad4a7b029e..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class UserPermission < Command - def help - return <<-EOF -Show all permissions for the particular user. -Syntax : user_permission
-For example: - - hbase> user_permission - hbase> user_permission 'table1' -EOF - end - - def command(table=nil) - #format_simple_command do - #admin.user_permission(table) - now = Time.now - formatter.header(["User", "Table,Family,Qualifier:Permission"]) - - count = security_admin.user_permission(table) do |user, permission| - formatter.row([ user, permission]) - end - - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb deleted file mode 100644 index 040ad7e7ef..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Whoami < Command - def help - return <<-EOF -Show the current hbase user. -Syntax : whoami -For example: - - hbase> whoami -EOF - end - - def command() - puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh deleted file mode 100755 index 89a214e5a8..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -if [ "$HBASE_MANAGES_ZK" = "" ]; then - HBASE_MANAGES_ZK=true -fi - -if [ "$HBASE_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-hbase-parent/titan-hbase-096/pom.xml b/titan-hbase-parent/titan-hbase-11/pom.xml similarity index 75% rename from titan-hbase-parent/titan-hbase-096/pom.xml rename to titan-hbase-parent/titan-hbase-11/pom.xml index 2ea2f7a92d..40be33c02e 100644 --- a/titan-hbase-parent/titan-hbase-096/pom.xml +++ b/titan-hbase-parent/titan-hbase-11/pom.xml @@ -6,8 +6,8 @@ 1.0.1-SNAPSHOT ../pom.xml - titan-hbase-096 - Titan-HBase: 0.96 Compatibility Shim + titan-hbase-11 + Titan-HBase: 1.1 Compatibility Shim http://thinkaurelius.github.com/titan/ @@ -30,7 +30,7 @@ org.apache.hbase hbase-client - ${hbase096.version} + ${hbase111.version} true @@ -50,7 +50,7 @@ org.apache.hbase hbase-server - ${hbase096.version} + ${hbase111.version} true @@ -133,52 +133,6 @@ - - hadoop1 - - - - hadoop.profile - 1.0 - - - - - ${hbase096.core.version}-hadoop1 - - - - - - maven-surefire-plugin - - - ${project.basedir}/conf - - - - - - - - - org.apache.hadoop - hadoop-core - - - org.apache.hadoop - hadoop-client - - - - io.netty - netty - - - - hadoop2 @@ -192,16 +146,19 @@ org.apache.hadoop hadoop-annotations + ${hadoop2.version} true org.apache.hadoop hadoop-hdfs + ${hadoop2.version} true org.apache.hadoop hadoop-client + ${hadoop2.version} true diff --git a/titan-hbase-parent/titan-hbase-11/readme.txt b/titan-hbase-parent/titan-hbase-11/readme.txt new file mode 100644 index 0000000000..4560298aa1 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/readme.txt @@ -0,0 +1 @@ +request to add this tree in main titan via pull request #1219 diff --git a/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java new file mode 100644 index 0000000000..86b928bb2c --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java @@ -0,0 +1,127 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.thinkaurelius.titan.util.system.IOUtils; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; + +public class HBaseAdmin1_1 implements AdminMask +{ + + private static final Logger log = LoggerFactory.getLogger(HBaseAdmin1_1.class); + + private final Admin adm; + + public HBaseAdmin1_1(HBaseAdmin adm) + { + this.adm = adm; + } + @Override + public void clearTable(String tableString, long timestamp) throws IOException + { + TableName tableName = TableName.valueOf(tableString); + + if (!adm.tableExists(tableName)) { + log.debug("Attempted to clear table {} before it exists (noop)", tableString); + return; + } + + if (!adm.isTableDisabled(tableName)) + adm.disableTable(tableName); + + if (!adm.isTableDisabled(tableName)) + throw new RuntimeException("Unable to disable table " + tableName); + + // This API call appears to both truncate and reenable the table. + log.info("Truncating table {}", tableName); + adm.truncateTable(tableName, true /* preserve splits */); + + try { + adm.enableTable(tableName); + } catch (TableNotDisabledException e) { + // This triggers seemingly every time in testing with 1.0.2. + log.debug("Table automatically reenabled by truncation: {}", tableName, e); + } + } + + @Override + public HTableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException + { + return adm.getTableDescriptor(TableName.valueOf(tableString)); + } + + @Override + public boolean tableExists(String tableString) throws IOException + { + return adm.tableExists(TableName.valueOf(tableString)); + } + + @Override + public void createTable(HTableDescriptor desc) throws IOException + { + adm.createTable(desc); + } + + @Override + public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException + { + adm.createTable(desc, startKey, endKey, numRegions); + } + + @Override + public int getEstimatedRegionServerCount() + { + int serverCount = -1; + try { + serverCount = adm.getClusterStatus().getServers().size(); + log.debug("Read {} servers from HBase ClusterStatus", serverCount); + } catch (IOException e) { + log.debug("Unable to retrieve HBase cluster status", e); + } + return serverCount; + } + + @Override + public void disableTable(String tableString) throws IOException + { + adm.disableTable(TableName.valueOf(tableString)); + } + + @Override + public void enableTable(String tableString) throws IOException + { + adm.enableTable(TableName.valueOf(tableString)); + } + + @Override + public boolean isTableDisabled(String tableString) throws IOException + { + return adm.isTableDisabled(TableName.valueOf(tableString)); + } + + @Override + public void addColumn(String tableString, HColumnDescriptor columnDescriptor) throws IOException + { + adm.addColumn(TableName.valueOf(tableString), columnDescriptor); + } + + @Override + public void close() throws IOException + { + adm.close(); + } +} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java similarity index 85% rename from titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java rename to titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java index 14e6376140..a25999f9d1 100644 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java @@ -6,11 +6,12 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.io.compress.Compression; -public class HBaseCompat0_96 implements HBaseCompat { +public class HBaseCompat1_1 implements HBaseCompat { @Override public void setCompression(HColumnDescriptor cd, String algo) { @@ -26,7 +27,7 @@ public HTableDescriptor newTableDescriptor(String tableName) { @Override public ConnectionMask createConnection(Configuration conf) throws IOException { - return new HConnection0_96(HConnectionManager.createConnection(conf)); + return new HConnection1_1(ConnectionFactory.createConnection(conf)); } @Override @@ -40,4 +41,5 @@ public void setTimestamp(Delete d, long timestamp) { d.setTimestamp(timestamp); } + } diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java similarity index 55% rename from titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java rename to titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java index ed55390b80..12df7f4686 100644 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java @@ -2,15 +2,16 @@ import java.io.IOException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -public class HConnection0_94 implements ConnectionMask +public class HConnection1_1 implements ConnectionMask { - private final HConnection cnx; + private final Connection cnx; - public HConnection0_94(HConnection cnx) + public HConnection1_1(Connection cnx) { this.cnx = cnx; } @@ -18,13 +19,13 @@ public HConnection0_94(HConnection cnx) @Override public TableMask getTable(String name) throws IOException { - return new HTable0_94(cnx.getTable(name)); + return new HTable1_1(cnx.getTable(TableName.valueOf(name))); } @Override public AdminMask getAdmin() throws IOException { - return new HBaseAdmin0_94(new HBaseAdmin(cnx)); + return new HBaseAdmin1_1(new HBaseAdmin(cnx)); } @Override diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java similarity index 81% rename from titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java rename to titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java index 719a48e136..02bc06e2db 100644 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java @@ -9,12 +9,13 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Row; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; -public class HTable0_94 implements TableMask +public class HTable1_1 implements TableMask { - private final HTableInterface table; + private final Table table; - public HTable0_94(HTableInterface table) + public HTable1_1(Table table) { this.table = table; } @@ -35,7 +36,7 @@ public Result[] get(List gets) throws IOException public void batch(List writes, Object[] results) throws IOException, InterruptedException { table.batch(writes, results); - table.flushCommits(); + /* table.flushCommits(); not needed anymore */ } @Override diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb new file mode 100755 index 0000000000..5bcb5b6b60 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Add or remove servers from draining mode via zookeeper + +require 'optparse' +include Java + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "draining_servers" + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: ./hbase org.jruby.Main #{NAME}.rb [options] add|remove|list || ..." + opts.separator 'Add remove or list servers in draining mode. Can accept either hostname to drain all region servers' + + 'in that host, a host:port pair or a host,port,startCode triplet. More than one server can be given separated by space' + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end +end +optparse.parse! + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +def getServerNames(hostOrServers, config) + ret = [] + + for hostOrServer in hostOrServers + # check whether it is already serverName. No need to connect to cluster + parts = hostOrServer.split(',') + if parts.size() == 3 + ret << hostOrServer + else + admin = HBaseAdmin.new(config) if not admin + servers = getServers(admin) + + hostOrServer = hostOrServer.gsub(/:/, ",") + for server in servers + ret << server if server.start_with?(hostOrServer) + end + end + end + + admin.close() if admin + return ret +end + +def addServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.createAndFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +def removeServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.deleteNodeFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +# list servers in draining mode +def listServers(options) + config = HBaseConfiguration.create() + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + servers.each {|server| puts server} +end + +hostOrServers = ARGV[1..ARGV.size()] + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase") + logger.setLevel(org.apache.log4j.Level::WARN) + logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper") + logger.setLevel(org.apache.log4j.Level::WARN) + end + return apacheLogger +end + +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'add' + if ARGV.length < 2 + puts optparse + exit 1 + end + addServers(options, hostOrServers) + when 'remove' + if ARGV.length < 2 + puts optparse + exit 1 + end + removeServers(options, hostOrServers) + when 'list' + listServers(options) + else + puts optparse + exit 3 +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh similarity index 63% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh index 37b5685085..98b27e31b5 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2011 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -24,13 +22,14 @@ # Turn off the balancer before running this script. function usage { echo "Usage: graceful_stop.sh [--config ] [-d] [-e] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " d|debug Print helpful debug information" - echo " hostname Hostname of server we are to stop" - echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" + echo " thrift If we should stop/start thrift before/after the hbase stop/start" + echo " rest If we should stop/start rest before/after the hbase stop/start" + echo " restart If we should restart after graceful stop" + echo " reload Move offloaded regions back on to the restarted server" + echo " d|debug Print helpful debug information" + echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." + echo " hostname Hostname of server we are to stop" + echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" exit 1 } @@ -48,6 +47,7 @@ reload= debug= thrift= rest= +maxthreads=1 failfast= while [ $# -gt 0 ] do @@ -58,6 +58,7 @@ do --reload) reload=true; shift;; --failfast | -e) failfast=true; shift;; --debug | -d) debug="--debug"; shift;; + --maxthreads) shift; maxthreads=$1; shift;; --) shift; break;; -*) usage ;; *) break;; # terminate while loop @@ -83,12 +84,19 @@ fi hostname=$1 filename="/tmp/$hostname" +local=false +localhostname=`/bin/hostname` + +if [ "$localhostname" == "$hostname" ]; then + local=true +fi + log "Disabling load balancer" HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` log "Previous balancer state was $HBASE_BALANCER_STATE" log "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname +HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads unload $hostname log "Unloaded $hostname region(s)" # Stop the server(s). Have to put hostname into its own little file for hbase-daemons.sh @@ -96,30 +104,53 @@ hosts="/tmp/$(basename $0).$$.tmp" echo $hostname >> $hosts if [ "$thrift" != "" ]; then log "Stopping thrift" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop thrift + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift + fi fi if [ "$rest" != "" ]; then log "Stopping rest" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop rest + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest + fi fi log "Stopping regionserver" -"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver - +if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop regionserver +else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver +fi if [ "$restart" != "" ]; then log "Restarting regionserver" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start regionserver + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver + fi if [ "$thrift" != "" ]; then log "Restarting thrift" # -b 0.0.0.0 says listen on all interfaces rather than just default. - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start thrift -b 0.0.0.0 + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 + fi fi if [ "$rest" != "" ]; then log "Restarting rest" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start rest + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest + fi fi if [ "$reload" != "" ]; then log "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname + HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads load $hostname log "Reloaded $hostname region(s)" fi fi @@ -127,7 +158,7 @@ fi # Restore balancer state if [ $HBASE_BALANCER_STATE != "false" ]; then log "Restoring balancer state to " $HBASE_BALANCER_STATE - log "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null + echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null fi # Cleanup tmp files. diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase similarity index 79% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hbase rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase index 6d59143674..6ad7a91097 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase @@ -1,8 +1,6 @@ #! /usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -34,8 +32,9 @@ # HBASE_CLASSPATH_PREFIX Extra Java CLASSPATH entries that should be # prefixed to the system classpath. # -# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. +# HBASE_HEAPSIZE The maximum amount of heap to use. +# Default is unset and uses the JVMs default setting +# (usually 1/4th of the available memory). # # HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding # native libraries. @@ -52,6 +51,9 @@ # JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. # Empty by default. # +# HBASE_SHELL_OPTS Extra options passed to the hbase shell. +# Empty by default. +# bin=`dirname "$0"` bin=`cd "$bin">/dev/null; pwd` @@ -63,11 +65,6 @@ case "`uname`" in CYGWIN*) cygwin=true;; esac -MVN="mvn" -if [ "$M2_HOME" != "" ]; then - MVN="${M2_HOME}"/bin/mvn -fi - # Detect if we are in hbase sources dir in_dev_env=false if [ -d "${HBASE_HOME}/target" ]; then @@ -85,7 +82,8 @@ if [ $# = 0 ]; then echo "Some commands take arguments. Pass no args or -h for usage." echo " shell Run the HBase shell" echo " hbck Run the hbase 'fsck' tool" - echo " hlog Write-ahead-log analyzer" + echo " snapshot Create a new snapshot of a table" + echo " wal Write-ahead-log analyzer" echo " hfile Store file analyzer" echo " zkcli Run the ZooKeeper shell" echo " upgrade Upgrade hbase" @@ -98,6 +96,8 @@ if [ $# = 0 ]; then echo " clean Run the HBase clean up script" echo " classpath Dump hbase CLASSPATH" echo " mapredcp Dump CLASSPATH entries required by mapreduce" + echo " pe Run PerformanceEvaluation" + echo " ltt Run LoadTestTool" echo " version Print the version" echo " CLASSNAME Run the class named CLASSNAME" exit 1 @@ -108,18 +108,29 @@ COMMAND=$1 shift JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m # override default settings for this command, if applicable if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" fi -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX +add_size_suffix() { + # add an 'm' suffix if the argument is missing one, otherwise use whats there + local val="$1" + local lastchar=${val: -1} + if [[ "mMgG" == *$lastchar* ]]; then + echo $val + else + echo ${val}m + fi +} + +if [[ -n "$HBASE_HEAPSIZE" ]]; then + JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_HEAPSIZE)" +fi + +if [[ -n "$HBASE_OFFHEAPSIZE" ]]; then + JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)" fi # so that filenames w/ spaces are handled correctly in loops below @@ -146,6 +157,17 @@ if [ -d "$HBASE_HOME/hbase-server/target/hbase-webapps" ]; then add_to_cp_if_exists "${HBASE_HOME}/hbase-server/target" fi +add_maven_deps_to_classpath() { + f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ] + then + echo "As this is a development environment, we need ${f} to be generated from maven (command: mvn install -DskipTests)" + exit 1 + fi + CLASSPATH=${CLASSPATH}:`cat "${f}"` +} + + add_maven_deps_to_classpath() { f="${HBASE_HOME}/target/cached_classpath.txt" if [ ! -f "${f}" ]; then @@ -164,7 +186,7 @@ add_maven_deps_to_classpath() { if [ -f "${optf}" ]; then MVN_OPTS=`cat "${optf}"` fi - "$MVN" -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null IFS= fi CLASSPATH=${CLASSPATH}:`cat "${f}"` @@ -200,13 +222,6 @@ if [ "$HBASE_LOGFILE" = "" ]; then HBASE_LOGFILE='hbase.log' fi -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - function append_path() { if [ -z "$1" ]; then echo $2 @@ -217,15 +232,40 @@ function append_path() { JAVA_PLATFORM="" +# if HBASE_LIBRARY_PATH is defined lets use it as first or second option +if [ "$HBASE_LIBRARY_PATH" != "" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") +fi + #If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) -if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") +# Allow this functionality to be disabled +if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then + HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) + if [ -f ${HADOOP_IN_PATH} ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") + fi + CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +fi + +# Add user-specified CLASSPATH last +if [ "$HBASE_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} +fi + +# Add user-specified CLASSPATH prefix first +if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then + CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH} +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HBASE_HOME=`cygpath -d "$HBASE_HOME"` + HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` fi if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then @@ -241,16 +281,6 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then fi fi -# Add user-specified CLASSPATH last -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} -fi - -# Add user-specified CLASSPATH prefix first -if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then - CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH} -fi - # cygwin path translation if $cygwin; then JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` @@ -286,14 +316,17 @@ if [ "$COMMAND" = "shell" ] ; then CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" elif [ "$COMMAND" = "hbck" ] ; then CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -elif [ "$COMMAND" = "hlog" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' +# TODO remove old 'hlog' version +elif [ "$COMMAND" = "hlog" -o "$COMMAND" = "wal" ] ; then + CLASS='org.apache.hadoop.hbase.wal.WALPrettyPrinter' elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFile' + CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter' elif [ "$COMMAND" = "zkcli" ] ; then CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer" elif [ "$COMMAND" = "upgrade" ] ; then CLASS="org.apache.hadoop.hbase.migration.UpgradeTo96" +elif [ "$COMMAND" = "snapshot" ] ; then + CLASS="org.apache.hadoop.hbase.snapshot.CreateSnapshot" elif [ "$COMMAND" = "master" ] ; then CLASS='org.apache.hadoop.hbase.master.HMaster' if [ "$1" != "stop" ] && [ "$1" != "clear" ] ; then @@ -345,6 +378,12 @@ elif [ "$COMMAND" = "mapredcp" ] ; then elif [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH exit 0 +elif [ "$COMMAND" = "pe" ] ; then + CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' + HBASE_OPTS="$HBASE_OPTS $HBASE_PE_OPTS" +elif [ "$COMMAND" = "ltt" ] ; then + CLASS='org.apache.hadoop.hbase.util.LoadTestTool' + HBASE_OPTS="$HBASE_OPTS $HBASE_LTT_OPTS" elif [ "$COMMAND" = "version" ] ; then CLASS='org.apache.hadoop.hbase.util.VersionInfo' else @@ -372,10 +411,11 @@ else HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" fi +HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" # Exec unless HBASE_NOEXEC is set. export CLASSPATH if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" + "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" else - exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" + exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh similarity index 88% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh index 8f1e4b479f..596e366a30 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh @@ -26,12 +26,12 @@ # Default is ${HADOOP_CONF_DIR}/regionservers # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. # -usage="Usage: hbase-cleanup.sh (zk|hdfs|all)" +usage="Usage: hbase-cleanup.sh (--cleanZk|--cleanHdfs|--cleanAll|--cleanAcls)" bin=`dirname "$0"` bin=`cd "$bin">/dev/null; pwd` @@ -40,7 +40,7 @@ bin=`cd "$bin">/dev/null; pwd` . "$bin"/hbase-config.sh case $1 in - --cleanZk|--cleanHdfs|--cleanAll) + --cleanZk|--cleanHdfs|--cleanAll|--cleanAcls) matches="yes" ;; *) ;; esac @@ -90,6 +90,11 @@ execute_hdfs_command() { "$bin"/hbase org.apache.hadoop.fs.FsShell $command 2>&1 } +execute_clean_acls() { + command=$1; + "$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZkAclReset $command 2>&1 +} + clean_up() { case $1 in --cleanZk) @@ -102,6 +107,9 @@ clean_up() { execute_zk_command "rmr ${zparent}"; execute_hdfs_command "-rmr ${hrootdir}" ;; + --cleanAcls) + execute_clean_acls; + ;; *) ;; esac diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd new file mode 100755 index 0000000000..5c1f1860f5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd @@ -0,0 +1,78 @@ +@rem/* +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem included in all the hbase scripts with source command +@rem should not be executable directly +@rem also should not be passed any arguments, since we need original $* +@rem Modelled after $HADOOP_HOME/bin/hadoop-env.sh. + +@rem Make sure java environment is set +@rem + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +if "%1" == "--config" ( + set HBASE_CONF_DIR=%2 + shift + shift +) + +@rem the root of the hbase installation +if not defined HBASE_HOME ( + set HBASE_HOME=%HBASE_BIN_PATH%\.. +) + +@rem Allow alternate hbase conf dir location. +if not defined HBASE_CONF_DIR ( + set HBASE_CONF_DIR=%HBASE_HOME%\conf +) + +@rem List of hbase regions servers. +if not defined HBASE_REGIONSERVERS ( + set HBASE_REGIONSERVERS=%HBASE_CONF_DIR%\regionservers +) + +@rem List of hbase secondary masters. +if not defined HBASE_BACKUP_MASTERS ( + set HBASE_BACKUP_MASTERS=%HBASE_CONF_DIR%\backup-masters +) + +@rem Source the hbase-env.sh. Will have JAVA_HOME defined. +if exist "%HBASE_CONF_DIR%\hbase-env.cmd" ( + call "%HBASE_CONF_DIR%\hbase-env.cmd" +) + +if not defined JAVA_HOME ( + echo Warning: JAVA_HOME environment variable is not set. Defaulting to c:\apps\java + set JAVA_HOME=c:\apps\java +) + +if not exist "%JAVA_HOME%\bin\java.exe" ( + echo Error: JAVA_HOME is incorrectly set or could not find java at the location %JAVA_HOME%\bin\ + exit /B 2 +) + +set JAVA="%JAVA_HOME%\bin\java" + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh similarity index 81% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh index 0300fc5ca3..5c518b1dc8 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh @@ -1,7 +1,5 @@ # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -76,7 +74,7 @@ HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" # List of hbase secondary masters. HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" # Thrift JMX opts -if [[ -n "$HBASE_JMX_OPTS" && -z "$HBASE_THRIFT_JMX_OPTS" ]]; then +if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_THRIFT_JMX_OPTS" ]; then HBASE_THRIFT_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10103" fi # Thrift opts @@ -85,7 +83,7 @@ if [ -z "$HBASE_THRIFT_OPTS" ]; then fi # REST JMX opts -if [[ -n "$HBASE_JMX_OPTS" && -z "$HBASE_REST_JMX_OPTS" ]]; then +if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_REST_JMX_OPTS" ]; then HBASE_REST_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10105" fi # REST opts @@ -122,34 +120,17 @@ fi # memory usage to explode. Tune the variable down to prevent vmem explosion. export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} +# Now having JAVA_HOME defined is required if [ -z "$JAVA_HOME" ]; then - for candidate in \ - /usr/lib/jvm/java-6-sun \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ - /usr/lib/j2sdk1.6-sun \ - /usr/java/jdk1.6* \ - /usr/java/jre1.6* \ - /Library/Java/Home ; do - if [ -e $candidate/bin/java ]; then - export JAVA_HOME=$candidate - break - fi - done - # if we didn't set it - if [ -z "$JAVA_HOME" ]; then cat 1>&2 < http://java.sun.com/javase/downloads/ < | +| > http://www.oracle.com/technetwork/java/javase/downloads | | | -| HBase requires Java 1.6 or later. | -| NOTE: This script will find Sun Java whether you install using the | -| binary or the RPM based installer. | +| HBase requires Java 1.7 or later. | +======================================================================+ EOF exit 1 - fi fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh similarity index 94% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh index caa74d8d7b..4ad657f137 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -144,8 +142,15 @@ fi JAVA=$JAVA_HOME/bin/java export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log + +if [ -z "${HBASE_ROOT_LOGGER}" ]; then export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"} +fi + +if [ -z "${HBASE_SECURITY_LOGGER}" ]; then export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"} +fi + logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" @@ -165,7 +170,7 @@ if [ "$HBASE_NICENESS" = "" ]; then export HBASE_NICENESS=0 fi -thiscmd=$0 +thiscmd="$bin/$(basename ${BASH_SOURCE-$0})" args=$@ case $startStop in @@ -214,17 +219,17 @@ case $startStop in #if the cluster is being stopped then do not restart it again. zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - zshutdown=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.state` - if [ "$zshutdown" == "null" ]; then zshutdown="shutdown"; fi - zFullShutdown=$zparent/$zshutdown - $bin/hbase zkcli stat $zFullShutdown 2>&1 | grep "Node does not exist" 1>/dev/null 2>&1 + zkrunning=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.state` + if [ "$zkrunning" == "null" ]; then zkrunning="running"; fi + zkFullRunning=$zparent/$zkrunning + $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep "Node does not exist" 1>/dev/null 2>&1 #grep returns 0 if it found something, 1 otherwise if [ $? -eq 0 ]; then exit 1 fi #If ZooKeeper cannot be found, then do not restart - $bin/hbase zkcli stat $zFullShutdown 2>&1 | grep Exception | grep ConnectionLoss 1>/dev/null 2>&1 + $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep Exception | grep ConnectionLoss 1>/dev/null 2>&1 if [ $? -eq 0 ]; then exit 1 fi @@ -248,7 +253,6 @@ case $startStop in echo "`date` Terminating $command" >> $loglog kill $pidToKill > /dev/null 2>&1 waitForProcessEnd $pidToKill $command - rm $pid else retval=$? echo no $command to stop because kill -0 of pid $pidToKill failed with status $retval @@ -256,6 +260,7 @@ case $startStop in else echo no $command to stop because no pid file $pid fi + rm -f $pid ;; (restart) diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh similarity index 96% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh index 843eaaa74f..21ce635777 100755 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby similarity index 100% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd new file mode 100755 index 0000000000..da7bcd9dea --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd @@ -0,0 +1,439 @@ +@echo off +@rem/* +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ +@rem +@rem The hbase command script. Based on the hadoop command script putting +@rem in hbase classes, libs and configurations ahead of hadoop's. +@rem +@rem TODO: Narrow the amount of duplicated code. +@rem +@rem Environment Variables: +@rem +@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +@rem +@rem HBASE_CLASSPATH Extra Java CLASSPATH entries. +@rem +@rem HBASE_HEAPSIZE The maximum amount of heap to use. +@rem Default is unset and uses the JVMs default setting +@rem (usually 1/4th of the available memory). +@rem +@rem HBASE_OPTS Extra Java runtime options. +@rem +@rem HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. +@rem +@rem HBASE_ROOT_LOGGER The root appender. Default is INFO,console +@rem +@rem JRUBY_HOME JRuby path: $JRUBY_HOME\lib\jruby.jar should exist. +@rem Defaults to the jar packaged with HBase. +@rem +@rem JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. +@rem Empty by default. +@rem HBASE_SHELL_OPTS Extra options passed to the hbase shell. +@rem Empty by default. + + +setlocal enabledelayedexpansion + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +rem This will set HBASE_HOME, etc. +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call "%hbase-config-script%" %* +if "%1" == "--config" ( + shift + shift +) + +rem Detect if we are in hbase sources dir +set in_dev_env=false + +if exist "%HBASE_HOME%\target" set in_dev_env=true + +rem --service is an internal option. used by MSI setup to install HBase as a windows service +if "%1" == "--service" ( + set service_entry=true + shift +) + +set hbase-command=%1 +shift + +@rem if no args specified, show usage +if "%hbase-command%"=="" ( + goto :print_usage + endlocal + goto :eof +) + +set JAVA_HEAP_MAX= +set JAVA_OFFHEAP_MAX= + +rem check envvars which might override default args +if defined HBASE_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%HBASE_HEAPSIZE%m +) + +if defined HBASE_OFFHEAPSIZE ( + set JAVA_OFFHEAP_MAX=-XX:MaxDirectMemory=%HBASE_OFFHEAPSIZE%m +) + +set CLASSPATH=%HBASE_CONF_DIR%;%JAVA_HOME%\lib\tools.jar + +rem Add maven target directory +set cached_classpath_filename=%HBASE_HOME%\target\cached_classpath.txt +if "%in_dev_env%"=="true" ( + + rem adding maven main classes to classpath + for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( + if exist %%i\target\classes set CLASSPATH=!CLASSPATH!;%%i\target\classes + ) + + rem adding maven test classes to classpath + rem For developers, add hbase classes to CLASSPATH + for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( + if exist %%i\target\test-classes set CLASSPATH=!CLASSPATH!;%%i\target\test-classes + ) + + if not exist "%cached_classpath_filename%" ( + echo "As this is a development environment, we need %cached_classpath_filename% to be generated from maven (command: mvn install -DskipTests)" + goto :eof + ) + + for /f "delims=" %%i in ('type "%cached_classpath_filename%"') do set CLASSPATH=%CLASSPATH%;%%i +) + +@rem For releases add hbase webapps to CLASSPATH +@rem Webapps must come first else it messes up Jetty +if exist "%HBASE_HOME%\hbase-webapps" ( + set CLASSPATH=%CLASSPATH%;%HBASE_HOME% +) + +if exist "%HBASE_HOME%\target\hbase-webapps" ( + set CLASSPATH=%CLASSPATH%;%HBASE_HOME%\target +) + +for /F %%f in ('dir /b "%HBASE_HOME%\hbase*.jar" 2^>nul') do ( + if not "%%f:~-11"=="sources.jar" ( + set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\%%f + ) +) + +@rem Add libs to CLASSPATH +if exist "%HBASE_HOME%\lib" ( + set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\lib\* +) + +@rem Add user-specified CLASSPATH last +if defined HBASE_CLASSPATH ( + set CLASSPATH=%CLASSPATH%;%HBASE_CLASSPATH% +) + +@rem Default log directory and file +if not defined HBASE_LOG_DIR ( + set HBASE_LOG_DIR=%HBASE_HOME%\logs +) + +if not defined HBASE_LOGFILE ( + set HBASE_LOGFILE=hbase.log +) + +set JAVA_PLATFORM= + +rem If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +set PATH=%PATH%;"%HADOOP_HOME%\bin" +set HADOOP_IN_PATH=hadoop.cmd + +if exist "%HADOOP_HOME%\bin\%HADOOP_IN_PATH%" ( + set hadoopCpCommand=call %HADOOP_IN_PATH% classpath 2^>nul + for /f "eol= delims=" %%i in ('!hadoopCpCommand!') do set CLASSPATH_FROM_HADOOP=%%i + if defined CLASSPATH_FROM_HADOOP ( + set CLASSPATH=%CLASSPATH%;!CLASSPATH_FROM_HADOOP! + ) + set HADOOP_CLASSPATH=%CLASSPATH% + + set hadoopJLPCommand=call %HADOOP_IN_PATH% org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2^>nul + for /f "eol= delims=" %%i in ('!hadoopJLPCommand!') do set HADOOP_JAVA_LIBRARY_PATH=%%i + if not defined JAVA_LIBRARY_PATH ( + set JAVA_LIBRARY_PATH=!HADOOP_JAVA_LIBRARY_PATH! + ) else ( + set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!HADOOP_JAVA_LIBRARY_PATH! + ) +) + +if exist "%HBASE_HOME%\build\native" ( + set platformCommand=call %JAVA% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName + for /f %%i in ('!platformCommand!') do set JAVA_PLATFORM=%%i + set _PATH_TO_APPEND=%HBASE_HOME%\build\native\!JAVA_PLATFORM!;%HBASE_HOME%\build\native\!JAVA_PLATFORM!\lib + if not defined JAVA_LIBRARY_PATH ( + set JAVA_LIBRARY_PATH=!_PATH_TO_APPEND! + ) else ( + set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!_PATH_TO_APPEND! + ) +) + +rem This loop would set %hbase-command-arguments% +set _hbasearguments= +:MakeCmdArgsLoop + if [%1]==[] goto :EndLoop + + if not defined _hbasearguments ( + set _hbasearguments=%1 + ) else ( + set _hbasearguments=!_hbasearguments! %1 + ) + shift +goto :MakeCmdArgsLoop +:EndLoop + +set hbase-command-arguments=%_hbasearguments% + +@rem figure out which class to run +set corecommands=shell master regionserver thrift thrift2 rest avro hlog wal hbck hfile zookeeper zkcli upgrade mapredcp +for %%i in ( %corecommands% ) do ( + if "%hbase-command%"=="%%i" set corecommand=true +) + +if defined corecommand ( + call :%hbase-command% %hbase-command-arguments% +) else ( + if "%hbase-command%" == "classpath" ( + echo %CLASSPATH% + goto :eof + ) + if "%hbase-command%" == "version" ( + set CLASS=org.apache.hadoop.hbase.util.VersionInfo + ) else ( + set CLASS=%hbase-command% + ) +) + +if not defined HBASE_IDENT_STRING ( + set HBASE_IDENT_STRING=%USERNAME% +) + +@rem Set the right GC options based on the what we are running +set servercommands=master regionserver thrift thrift2 rest avro zookeeper +for %%i in ( %servercommands% ) do ( + if "%hbase-command%"=="%%i" set servercommand=true +) + +if "%servercommand%" == "true" ( + set HBASE_OPTS=%HBASE_OPTS% %SERVER_GC_OPTS% +) else ( + set HBASE_OPTS=%HBASE_OPTS% %CLIENT_GC_OPTS% +) + +@rem If HBase is run as a windows service, configure logging +if defined service_entry ( + set HBASE_LOG_PREFIX=hbase-%hbase-command%-%COMPUTERNAME% + set HBASE_LOGFILE=!HBASE_LOG_PREFIX!.log + if not defined HBASE_ROOT_LOGGER ( + set HBASE_ROOT_LOGGER=INFO,DRFA + ) + set HBASE_SECURITY_LOGGER=INFO,DRFAS + set loggc=!HBASE_LOG_DIR!\!HBASE_LOG_PREFIX!.gc + set loglog=!HBASE_LOG_DIR!\!HBASE_LOGFILE! + + if "%HBASE_USE_GC_LOGFILE%" == "true" ( + set HBASE_OPTS=%HBASE_OPTS% -Xloggc:"!loggc!" + ) +) + + +@rem Have JVM dump heap if we run out of memory. Files will be 'launch directory' +@rem and are named like the following: java_pid21612.hprof. Apparently it does not +@rem 'cost' to have this flag enabled. Its a 1.6 flag only. See: +@rem http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.dir="%HBASE_LOG_DIR%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.file="%HBASE_LOGFILE%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.home.dir="%HBASE_HOME%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.id.str="%HBASE_IDENT_STRING%" +set HBASE_OPTS=%HBASE_OPTS% -XX:OnOutOfMemoryError="taskkill /F /PID %p" + +if not defined HBASE_ROOT_LOGGER ( + set HBASE_ROOT_LOGGER=INFO,console +) +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger="%HBASE_ROOT_LOGGER%" + +if defined JAVA_LIBRARY_PATH ( + set HBASE_OPTS=%HBASE_OPTS% -Djava.library.path="%JAVA_LIBRARY_PATH%" +) + +rem Enable security logging on the master and regionserver only +if not defined HBASE_SECURITY_LOGGER ( + set HBASE_SECURITY_LOGGER=INFO,NullAppender + if "%hbase-command%"=="master" ( + set HBASE_SECURITY_LOGGER=INFO,DRFAS + ) + if "%hbase-command%"=="regionserver" ( + set HBASE_SECURITY_LOGGER=INFO,DRFAS + ) +) +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%" + +set HEAP_SETTINGS=%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX% +set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments% + +if defined service_entry ( + call :makeServiceXml %java_arguments% +) else ( + call %JAVA% %java_arguments% +) + +endlocal +goto :eof + +:shell + rem eg export JRUBY_HOME=/usr/local/share/jruby + if defined JRUBY_HOME ( + set CLASSPATH=%CLASSPATH%;%JRUBY_HOME%\lib\jruby.jar + set HBASE_OPTS=%HBASE_OPTS% -Djruby.home="%JRUBY_HOME%" -Djruby.lib="%JRUBY_HOME%\lib" + ) + rem find the hbase ruby sources + if exist "%HBASE_HOME%\lib\ruby" ( + set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\lib\ruby" + ) else ( + set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\hbase-shell\src\main\ruby" + ) + set HBASE_OPTS=%HBASE_OPTS% %HBASE_SHELL_OPTS% + + set CLASS=org.jruby.Main -X+O %JRUBY_OPTS% "%HBASE_HOME%\bin\hirb.rb" + goto :eof + +:master + set CLASS=org.apache.hadoop.hbase.master.HMaster + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_MASTER_OPTS% + ) + goto :eof + +:regionserver + set CLASS=org.apache.hadoop.hbase.regionserver.HRegionServer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_REGIONSERVER_OPTS% + ) + goto :eof + +:thrift + set CLASS=org.apache.hadoop.hbase.thrift.ThriftServer + if NOT "%1" == "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% + ) + goto :eof + +:thrift2 + set CLASS=org.apache.hadoop.hbase.thrift2.ThriftServer + if NOT "%1" == "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% + ) + goto :eof + +:rest + set CLASS=org.apache.hadoop.hbase.rest.RESTServer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_REST_OPTS% + ) + goto :eof + +:avro + set CLASS=org.apache.hadoop.hbase.avro.AvroServer + if NOT "%1"== "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_AVRO_OPTS% + ) + goto :eof + +:zookeeper + set CLASS=org.apache.hadoop.hbase.zookeeper.HQuorumPeer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_ZOOKEEPER_OPTS% + ) + goto :eof + +:hbck + set CLASS=org.apache.hadoop.hbase.util.HBaseFsck + goto :eof + +@rem TODO remove older 'hlog' command +:hlog + set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter + goto :eof + +:wal + set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter + goto :eof + +:hfile + set CLASS=org.apache.hadoop.hbase.io.hfile.HFile + goto :eof + +:zkcli + set CLASS=org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer + goto :eof + +:upgrade + set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96 + goto :eof + +:mapredcp + set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool + goto :eof + +:makeServiceXml + set arguments=%* + @echo ^ + @echo ^%hbase-command%^ + @echo ^%hbase-command%^ + @echo ^This service runs Isotope %hbase-command%^ + @echo ^%JAVA%^ + @echo ^%arguments%^ + @echo ^ + goto :eof + +:print_usage + echo Usage: hbase [^] ^ [^] + echo where ^ an option from one of these categories:: + echo Options: + echo --config DIR Configuration direction to use. Default: ./conf + echo. + echo Commands: + echo Some commands take arguments. Pass no args or -h for usage." + echo shell Run the HBase shell + echo hbck Run the hbase 'fsck' tool + echo wal Write-ahead-log analyzer + echo hfile Store file analyzer + echo zkcli Run the ZooKeeper shell + echo upgrade Upgrade hbase + echo master Run an HBase HMaster node + echo regionserver Run an HBase HRegionServer node + echo zookeeper Run a Zookeeper server + echo rest Run an HBase REST server + echo thrift Run the HBase Thrift server + echo thrift2 Run the HBase Thrift2 server + echo classpath Dump hbase CLASSPATH + echo mapredcp Dump CLASSPATH entries required by mapreduce + echo version Print the version + echo CLASSNAME Run the class named CLASSNAME + goto :eof diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb similarity index 98% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb index dc1feccfae..0503c29706 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb @@ -78,6 +78,7 @@ log_level = org.apache.log4j.Level::DEBUG $fullBackTrace = true @shell_debug = true + found.push(arg) puts "Setting DEBUG log level..." else # Presume it a script. Save it off for running later below @@ -91,6 +92,10 @@ # Delete all processed args found.each { |arg| ARGV.delete(arg) } +# Make sure debug flag gets back to IRB +if @shell_debug + ARGV.unshift('-d') +end # Set logging level to avoid verboseness org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-master-backup.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-master-backup.sh similarity index 85% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/local-master-backup.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/local-master-backup.sh index 1690a70f10..bd265abedf 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-master-backup.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-master-backup.sh @@ -1,7 +1,5 @@ -#!/bin/sh +#!/usr/bin/env bash #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -42,8 +40,9 @@ run_master () { DN=$2 export HBASE_IDENT_STRING="$USER-$DN" HBASE_MASTER_ARGS="\ - -D hbase.master.port=`expr 60000 + $DN` \ - -D hbase.master.info.port=`expr 60010 + $DN` \ + -D hbase.master.info.port=`expr 16010 + $DN` \ + -D hbase.regionserver.port=`expr 16020 + $DN` \ + -D hbase.regionserver.info.port=`expr 16030 + $DN` \ --backup" "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 master $HBASE_MASTER_ARGS } @@ -53,5 +52,9 @@ shift; for i in $* do - run_master $cmd $i + if [[ "$i" =~ ^[0-9]+$ ]]; then + run_master $cmd $i + else + echo "Invalid argument" + fi done diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh similarity index 87% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh index bf4372e21d..78b4195811 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh @@ -1,7 +1,5 @@ -#!/bin/sh +#!/usr/bin/env bash #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -42,8 +40,8 @@ run_regionserver () { DN=$2 export HBASE_IDENT_STRING="$USER-$DN" HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 60200 + $DN` \ - -D hbase.regionserver.info.port=`expr 60300 + $DN`" + -D hbase.regionserver.port=`expr 16200 + $DN` \ + -D hbase.regionserver.info.port=`expr 16300 + $DN`" "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 regionserver $HBASE_REGIONSERVER_ARGS } @@ -52,5 +50,9 @@ shift; for i in $* do - run_regionserver $cmd $i + if [[ "$i" =~ ^[0-9]+$ ]]; then + run_regionserver $cmd $i + else + echo "Invalid argument" + fi done diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh similarity index 91% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh index 114757c763..feca4ab865 100755 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2010 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -28,8 +26,8 @@ # Default is ${HBASE_CONF_DIR}/backup-masters # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. # # Modelled after $HADOOP_HOME/bin/slaves.sh. diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb similarity index 64% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb index cc57456f61..134b032b87 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb @@ -20,9 +20,8 @@ # not move a new region until successful confirm of region loading in new # location. Presumes balancer is disabled when we run (not harmful if its # on but this script and balancer will end up fighting each other). -# Does not work for case of multiple regionservers all running on the -# one node. require 'optparse' +require File.join(File.dirname(__FILE__), 'thread-pool') include Java import org.apache.hadoop.hbase.HConstants import org.apache.hadoop.hbase.HBaseConfiguration @@ -32,6 +31,8 @@ import org.apache.hadoop.hbase.client.HTable import org.apache.hadoop.hbase.client.HConnectionManager import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.util.Writables import org.apache.hadoop.conf.Configuration @@ -44,46 +45,10 @@ # Name of this script NAME = "region_mover" -# Get meta table reference -def getMetaTable(config) - # Keep meta reference in ruby global - if not $META - $META = HTable.new(config, HConstants::META_TABLE_NAME) - end - return $META -end - -# Get table instance. -# Maintains cache of table instances. -def getTable(config, name) - # Keep dictionary of tables in ruby global - if not $TABLES - $TABLES = {} - end - key = name.toString() - if not $TABLES[key] - $TABLES[key] = HTable.new(config, name) - end - return $TABLES[key] -end - -def closeTables() - if not $TABLES - return - end - - $LOG.info("Close all tables") - $TABLES.each do |name, table| - $TABLES.delete(name) - table.close() - end -end - -# Returns true if passed region is still on 'original' when we look at .META. +# Returns true if passed region is still on 'original' when we look at hbase:meta. def isSameServer(admin, r, original) server = getServerNameForRegion(admin, r) - return false unless server - return true unless original + return false unless server and original return server == original end @@ -94,56 +59,65 @@ def abort(why, e) end end -# Get servername that is up in .META.; this is hostname + port + startcode comma-delimited. +# Get servername that is up in hbase:meta; this is hostname + port + startcode comma-delimited. # Can return nil def getServerNameForRegion(admin, r) + return nil unless admin.isTableEnabled(r.getTableName) if r.isMetaRegion() # Hack zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(admin.getConfiguration(), "region_mover", nil) + mtl = org.apache.hadoop.hbase.zookeeper.MetaTableLocator.new() begin - tracker = org.apache.hadoop.hbase.zookeeper.MetaRegionTracker.new(zkw, RubyAbortable.new()) - tracker.start() - while not tracker.isLocationAvailable() + while not mtl.isLocationAvailable(zkw) sleep 0.1 end # Make a fake servername by appending ',' - metaServer = tracker.getMetaRegionLocation().toString() + "," - tracker.stop() + metaServer = mtl.getMetaRegionLocation(zkw).toString() + "," return metaServer ensure zkw.close() end end - table = nil - table = getMetaTable(admin.getConfiguration()) - g = Get.new(r.getRegionName()) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - result = table.get(g) - server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - return nil unless server - return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s + table = HTable.new(admin.getConfiguration(), HConstants::META_TABLE_NAME) + begin + g = Get.new(r.getRegionName()) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + result = table.get(g) + return nil unless result + server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + return nil unless server + return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s + ensure + table.close() + end end # Trys to scan a row from passed region # Throws exception if can't def isSuccessfulScan(admin, r) - scan = Scan.new(r.getStartKey()) + scan = Scan.new(r.getStartKey(), r.getStartKey()) scan.setBatch(1) scan.setCaching(1) - scan.setFilter(FirstKeyOnlyFilter.new()) - table = getTable(admin.getConfiguration(), r.getTableName()) - scanner = table.getScanner(scan) + scan.setFilter(FilterList.new(FirstKeyOnlyFilter.new(),InclusiveStopFilter.new(r.getStartKey()))) begin - results = scanner.next() - # We might scan into next region, this might be an empty table. - # But if no exception, presume scanning is working. + table = HTable.new(admin.getConfiguration(), r.getTableName()) + scanner = table.getScanner(scan) + begin + results = scanner.next() + # We might scan into next region, this might be an empty table. + # But if no exception, presume scanning is working. + ensure + scanner.close() + end + rescue org.apache.hadoop.hbase.TableNotFoundException, + org.apache.hadoop.hbase.TableNotEnabledException => e + $LOG.warn("Region " + r.getEncodedName() + " belongs to recently " + + "deleted/disabled table. Skipping... " + e.message) + return ensure - scanner.close() - # Do not close the htable. It is cached in $TABLES and - # may be reused in moving another region of same table. - # table.close() + table.close() unless table.nil? end end @@ -153,6 +127,7 @@ def move(admin, r, newServer, original) # Now move it. Do it in a loop so can retry if fail. Have seen issue where # we tried move region but failed and retry put it back on old location; # retry in this case. + retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) count = 0 same = true @@ -164,7 +139,8 @@ def move(admin, r, newServer, original) count = count + 1 begin admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) - rescue java.lang.reflect.UndeclaredThrowableException => e + rescue java.lang.reflect.UndeclaredThrowableException, + org.apache.hadoop.hbase.UnknownRegionException => e $LOG.info("Exception moving " + r.getEncodedName() + "; split/moved? Continuing: " + e) return @@ -185,15 +161,9 @@ def move(admin, r, newServer, original) java.lang.String.format("%.3f", (Time.now - start))) end -# Return the hostname portion of a servername (all up to first ',') -def getHostnamePortFromServerName(serverName) - parts = serverName.split(',') - return parts[0] + ":" + parts[1] -end - -# Return the hostname:port out of a servername (all up to first ',') -def getHostnameFromServerName(serverName) - return serverName.split(',')[0] +# Return the hostname:port out of a servername (all up to second ',') +def getHostPortFromServerName(serverName) + return serverName.split(',')[0..1] end # Return array of servernames where servername is hostname+port+startcode @@ -209,16 +179,17 @@ def getServers(admin) # Remove the servername whose hostname portion matches from the passed # array of servers. Returns as side-effect the servername removed. -def stripServer(servers, hostname) +def stripServer(servers, hostname, port) count = servers.length servername = nil for server in servers - if getHostnameFromServerName(server) == hostname + hostFromServerName, portFromServerName = getHostPortFromServerName(server) + if hostFromServerName == hostname and portFromServerName == port servername = servers.delete(server) end end # Check server to exclude is actually present - raise RuntimeError, "Server %s not online" % hostname unless servers.length < count + raise RuntimeError, "Server %s:%d not online" % [hostname, port] unless servers.length < count return servername end @@ -226,22 +197,25 @@ def stripServer(servers, hostname) # matches from the passed array of servers. def stripExcludes(servers, excludefile) excludes = readExcludes(excludefile) - servers = servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) } + servers = servers.find_all{|server| + !excludes.contains(getHostPortFromServerName(server).join(":")) + } # return updated servers list return servers end -# Return servername that matches passed hostname -def getServerName(servers, hostname) +# Return servername that matches passed hostname and port +def getServerName(servers, hostname, port) servername = nil for server in servers - if getHostnameFromServerName(server) == hostname + hostFromServerName, portFromServerName = getHostPortFromServerName(server) + if hostFromServerName == hostname and portFromServerName == port servername = server break end end - raise ArgumentError, "Server %s not online" % hostname unless servername + raise ArgumentError, "Server %s:%d not online" % [hostname, port] unless servername return servername end @@ -259,7 +233,7 @@ def configureLogging(options) # Get configuration instance def getConfiguration() config = HBaseConfiguration.create() - # No prefetching on .META. + # No prefetching on hbase:meta This is for versions pre 0.99. Newer versions do not prefetch. config.setInt("hbase.client.prefetch.limit", 1) # Make a config that retries at short intervals many times config.setInt("hbase.client.pause", 500) @@ -270,7 +244,7 @@ def getConfiguration() # Now get list of regions on targetServer def getRegions(config, servername) connection = HConnectionManager::getConnection(config); - return ProtobufUtil::getOnlineRegions(connection.getAdmin(ServerName.new(servername))); + return ProtobufUtil::getOnlineRegions(connection.getAdmin(ServerName.valueOf(servername))); end def deleteFile(filename) @@ -312,43 +286,54 @@ def readFile(filename) return regions end -# Move regions off the passed hostname -def unloadRegions(options, hostname) - # Get configuration - config = getConfiguration() +# Move regions off the passed hostname:port +def unloadRegions(options, hostname, port) # Clean up any old files. - filename = getFilename(options, hostname) + filename = getFilename(options, hostname, port) deleteFile(filename) + # Get configuration + config = getConfiguration() # Get an admin instance - admin = HBaseAdmin.new(config) + admin = HBaseAdmin.new(config) servers = getServers(admin) # Remove the server we are unloading from from list of servers. # Side-effect is the servername that matches this hostname - servername = stripServer(servers, hostname) + servername = stripServer(servers, hostname, port) # Remove the servers in our exclude list from list of servers. servers = stripExcludes(servers, options[:excludesFile]) puts "Valid region move targets: ", servers - movedRegions = java.util.ArrayList.new() + if servers.length == 0 + puts "No regions were moved - there was no server available" + exit 4 + end + movedRegions = java.util.Collections.synchronizedList(java.util.ArrayList.new()) while true rs = getRegions(config, servername) + # Remove those already tried to move + rs.removeAll(movedRegions) break if rs.length == 0 - count = 0 $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + - " during this cycle"); - for r in rs - # Get a random server to move the region to. - server = servers[rand(servers.length)] - $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + - count.to_s + " of " + rs.length.to_s + ") from server=" + - servername + " to server=" + server); - count = count + 1 - # Assert we can scan region in its current location - isSuccessfulScan(admin, r) - # Now move it. - move(admin, r, server, servername) - movedRegions.add(r) + " on " + servers.length.to_s + " servers using " + options[:maxthreads].to_s + " threads.") + counter = 0 + pool = ThreadPool.new(options[:maxthreads]) + server_index = 0 + while counter < rs.length do + pool.launch(rs,counter,server_index) do |_rs,_counter,_server_index| + $LOG.info("Moving region " + _rs[_counter].getEncodedName() + " (" + (_counter + 1).to_s + + " of " + _rs.length.to_s + ") to server=" + servers[_server_index] + " for " + servername) + # Assert we can scan region in its current location + isSuccessfulScan(admin, _rs[_counter]) + # Now move it. + move(admin, _rs[_counter], servers[_server_index], servername) + movedRegions.add(_rs[_counter]) + end + counter += 1 + server_index = (server_index + 1) % servers.length end + $LOG.info("Waiting for the pool to complete") + pool.stop + $LOG.info("Pool completed") end if movedRegions.size() > 0 # Write out file of regions moved @@ -358,12 +343,12 @@ def unloadRegions(options, hostname) end # Move regions to the passed hostname -def loadRegions(options, hostname) +def loadRegions(options, hostname, port) # Get configuration config = getConfiguration() # Get an admin instance admin = HBaseAdmin.new(config) - filename = getFilename(options, hostname) + filename = getFilename(options, hostname, port) regions = readFile(filename) return if regions.isEmpty() servername = nil @@ -373,18 +358,20 @@ def loadRegions(options, hostname) while Time.now < maxWait servers = getServers(admin) begin - servername = getServerName(servers, hostname) + servername = getServerName(servers, hostname, port) rescue ArgumentError => e - $LOG.info("hostname=" + hostname.to_s + " is not up yet, waiting"); + $LOG.info("hostname=" + hostname.to_s + ":" + port.to_s + " is not up yet, waiting"); end break if servername sleep 0.5 end $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) - count = 0 # sleep 20s to make sure the rs finished initialization. sleep 20 - for r in regions + counter = 0 + pool = ThreadPool.new(options[:maxthreads]) + while counter < regions.length do + r = regions[counter] exists = false begin isSuccessfulScan(admin, r) @@ -392,19 +379,23 @@ def loadRegions(options, hostname) rescue org.apache.hadoop.hbase.NotServingRegionException => e $LOG.info("Failed scan of " + e.message) end - count = count + 1 next unless exists currentServer = getServerNameForRegion(admin, r) if currentServer and currentServer == servername - $LOG.info("Region " + r.getRegionNameAsString() + " (" + count.to_s + - " of " + regions.length.to_s + ") already on target server=" + servername) + $LOG.info("Region " + r.getRegionNameAsString() + " (" + counter.to_s + + " of " + regions.length.to_s + ") already on target server=" + servername) + counter = counter + 1 next end - $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + - count.to_s + " of " + regions.length.to_s + ") from server=" + - currentServer + " to server=" + servername); - move(admin, r, servername, currentServer) + pool.launch(r,currentServer,counter) do |_r,_currentServer,_counter| + $LOG.info("Moving region " + _r.getRegionNameAsString() + " (" + (_counter + 1).to_s + + " of " + regions.length.to_s + ") from " + _currentServer.to_s + " to server=" + + servername); + move(admin, _r, servername, _currentServer) + end + counter = counter + 1 end + pool.stop end # Returns an array of hosts to exclude as region move targets @@ -430,10 +421,10 @@ def readExcludes(filename) return excludes end -def getFilename(options, targetServer) +def getFilename(options, targetServer, port) filename = options[:file] if not filename - filename = "/tmp/" + ENV['USER'] + targetServer + filename = "/tmp/" + ENV['USER'] + targetServer + ":" + port end return filename end @@ -442,10 +433,11 @@ def getFilename(options, targetServer) # Do command-line parsing options = {} optparse = OptionParser.new do |opts| - opts.banner = "Usage: #{NAME}.rb [options] load|unload " + opts.banner = "Usage: #{NAME}.rb [options] load|unload [|]" opts.separator 'Load or unload regions by moving one at a time' options[:file] = nil - opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| + options[:maxthreads] = 1 + opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| options[:file] = file end opts.on('-h', '--help', 'Display usage information') do @@ -459,6 +451,9 @@ def getFilename(options, targetServer) opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| options[:excludesFile] = file end + opts.on('-m', '--maxthreads=XX', 'Define the maximum number of threads to use to unload and reload the regions') do |number| + options[:maxthreads] = number.to_i + end end optparse.parse! @@ -467,21 +462,27 @@ def getFilename(options, targetServer) puts optparse exit 1 end -hostname = ARGV[1] +hostname, port = ARGV[1].split(":") if not hostname opts optparse exit 2 end + +# Get configuration +config = getConfiguration() +if not port + port = config.getInt(HConstants::REGIONSERVER_PORT, HConstants::DEFAULT_REGIONSERVER_PORT) +end +port = port.to_s + # Create a logger and save it to ruby global $LOG = configureLogging(options) case ARGV[0] when 'load' - loadRegions(options, hostname) + loadRegions(options, hostname, port) when 'unload' - unloadRegions(options, hostname) + unloadRegions(options, hostname, port) else puts optparse exit 3 end - -closeTables() diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb similarity index 96% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb index df35ec6237..878d58ab5d 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb @@ -64,7 +64,7 @@ org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) config = HBaseConfiguration.create -config.set 'fs.default.name', config.get(HConstants::HBASE_DIR) +config.set 'fs.defaultFS', config.get(HConstants::HBASE_DIR) # wait until the master is running admin = nil @@ -90,8 +90,8 @@ (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes ) end -scan.cache_blocks = false -scan.caching = 10 +scan.setCacheBlocks(false) +scan.setCaching(10) scan.setFilter(FirstKeyOnlyFilter.new) INFO = 'info'.to_java_bytes REGION_INFO = 'regioninfo'.to_java_bytes @@ -125,7 +125,7 @@ end scanner.close # If we're trying to see the status of all HBase tables, we need to include the -# .META. table, that is not included in our scan +# hbase:meta table, that is not included in our scan if $tablename.nil? meta_count += 1 end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh similarity index 72% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh index 9759f2b00c..af26ba95d7 100755 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -28,8 +26,8 @@ # Default is ${HADOOP_CONF_DIR}/regionservers # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. # # Modelled after $HADOOP_HOME/bin/slaves.sh. @@ -59,17 +57,22 @@ if [ "$HOSTLIST" = "" ]; then fi fi -for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi -done +regionservers=`cat "$HOSTLIST"` +if [ "$regionservers" = "localhost" ]; then + "$bin"/local-regionservers.sh start 1 +else + for regionserver in `cat "$HOSTLIST"`; do + if ${HBASE_SLAVE_PARALLEL:-true}; then + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" & + else # run each command serially + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" + fi + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi wait diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh similarity index 85% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh index 97843e8ed4..b3ac67f050 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -28,13 +26,17 @@ # Default is ${HADOOP_CONF_DIR}/regionservers # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. # # Modelled after $HADOOP_HOME/bin/slaves.sh. -usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" +usage_str="Usage: `basename $0` [--config ] [--rs-only] [--master-only] [--graceful] [--maxthreads xx]" + +function usage() { + echo "${usage_str}" +} bin=`dirname "$0"` bin=`cd "$bin">/dev/null; pwd` @@ -48,35 +50,42 @@ then exit $errCode fi -function usage() { - echo $usage - exit 1 -} - - RR_RS=1 RR_MASTER=1 RR_GRACEFUL=0 +RR_MAXTHREADS=1 -for x in "$@" ; do - case "$x" in +while [ $# -gt 0 ]; do + case "$1" in --rs-only|-r) RR_RS=1 RR_MASTER=0 RR_GRACEFUL=0 + shift ;; --master-only) RR_RS=0 RR_MASTER=1 RR_GRACEFUL=0 + shift ;; --graceful) RR_RS=0 RR_MASTER=0 RR_GRACEFUL=1 + shift + ;; + --maxthreads) + shift + RR_MAXTHREADS=$1 + shift + ;; + --help|-h) + usage + exit 0 ;; *) - echo Bad argument: $x + echo Bad argument: $1 usage exit 1 ;; @@ -85,7 +94,7 @@ done # quick function to get a value from the HBase config file # HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +distMode=`HBASE_CONF_DIR=${HBASE_CONF_DIR} $bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` if [ "$distMode" == 'false' ]; then if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then echo Cant do selective rolling restart if not running distributed @@ -158,7 +167,7 @@ else rs_parts=(${rs//,/ }) hostname=${rs_parts[0]} echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" + "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug --maxthreads "${RR_MAXTHREADS}" "$hostname" sleep 1 done fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb similarity index 83% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb index 87512bf88b..304afaf014 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb @@ -18,8 +18,8 @@ # # HBase ruby classes. -# Has wrapper classes for org.apache.hadoop.hbase.client.HBaseAdmin -# and for org.apache.hadoop.hbase.client.HTable. Classes take +# Has wrapper classes for org.apache.hadoop.hbase.client.Admin +# and for org.apache.hadoop.hbase.client.Table. Classes take # Formatters on construction and outputs any results using # Formatter methods. These classes are only really for use by # the hirb.rb HBase Shell script; they don't make much sense elsewhere. @@ -48,6 +48,7 @@ module HBaseConstants METHOD = "METHOD" MAXLENGTH = "MAXLENGTH" CACHE_BLOCKS = "CACHE_BLOCKS" + REVERSED = "REVERSED" REPLICATION_SCOPE = "REPLICATION_SCOPE" INTERVAL = 'INTERVAL' CACHE = 'CACHE' @@ -56,7 +57,19 @@ module HBaseConstants SPLITS_FILE = 'SPLITS_FILE' SPLITALGO = 'SPLITALGO' NUMREGIONS = 'NUMREGIONS' + REGION_REPLICATION = 'REGION_REPLICATION' + REGION_REPLICA_ID = 'REGION_REPLICA_ID' CONFIGURATION = org.apache.hadoop.hbase.HConstants::CONFIGURATION + ATTRIBUTES="ATTRIBUTES" + VISIBILITY="VISIBILITY" + AUTHORIZATIONS = "AUTHORIZATIONS" + SKIP_FLUSH = 'SKIP_FLUSH' + CONSISTENCY = "CONSISTENCY" + ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME' + CLUSTER_KEY = 'CLUSTER_KEY' + TABLE_CFS = 'TABLE_CFS' + CONFIG = 'CONFIG' + DATA = 'DATA' # Load constants from hbase java API def self.promote_constants(constants) @@ -78,3 +91,4 @@ def self.promote_constants(constants) require 'hbase/table' require 'hbase/replication_admin' require 'hbase/security' +require 'hbase/visibility_labels' diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb similarity index 73% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb index 8ab2ef4efb..a8f543726d 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb @@ -22,6 +22,9 @@ java_import org.apache.hadoop.hbase.util.Pair java_import org.apache.hadoop.hbase.util.RegionSplitter java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.ServerName +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -29,21 +32,20 @@ module Hbase class Admin include HBaseConstants - def initialize(configuration, formatter) - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - connection = @admin.getConnection() - @conf = configuration - @zk_wrapper = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(configuration, - "admin", nil) - zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() - @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) + def initialize(admin, formatter) + @admin = admin + @connection = @admin.getConnection() @formatter = formatter end + def close + @admin.close + end + #---------------------------------------------------------------------------------------------- # Returns a list of tables in hbase def list(regex = ".*") - @admin.getTableNames(regex).to_a + @admin.listTables(regex).map { |t| t.getNameAsString } end #---------------------------------------------------------------------------------------------- @@ -63,6 +65,11 @@ def compact(table_or_region_name, family = nil) end end + # Requests to compact all regions on the regionserver + def compact_regionserver(servername, major = false) + @admin.compactRegionServer(ServerName.valueOf(servername), major) + end + #---------------------------------------------------------------------------------------------- # Requests a table or region or column family major compaction def major_compact(table_or_region_name, family = nil) @@ -75,10 +82,12 @@ def major_compact(table_or_region_name, family = nil) end #---------------------------------------------------------------------------------------------- - # Requests a regionserver's HLog roll - def hlog_roll(server_name) - @admin.rollHLogWriter(server_name) + # Requests a regionserver's WAL roll + def wal_roll(server_name) + @admin.rollWALWriter(ServerName.valueOf(server_name)) end + # TODO remove older hlog_roll version + alias :hlog_roll :wal_roll #---------------------------------------------------------------------------------------------- # Requests a table or region split @@ -159,7 +168,7 @@ def disable_all(regex) #--------------------------------------------------------------------------------------------- # Throw exception if table doesn't exist def tableExists(table_name) - raise ArgumentError, "Table #{table_name} does not exist.'" unless exists?(table_name) + raise ArgumentError, "Table #{table_name} does not exist." unless exists?(table_name) end #---------------------------------------------------------------------------------------------- @@ -172,9 +181,9 @@ def disabled?(table_name) # Drops a table def drop(table_name) tableExists(table_name) - raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name) + raise ArgumentError, "Table #{table_name} is enabled. Disable it first." if enabled?(table_name) - @admin.deleteTable(table_name) + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) end #---------------------------------------------------------------------------------------------- @@ -188,6 +197,12 @@ def drop_all(regex) #---------------------------------------------------------------------------------------------- # Returns ZooKeeper status dump def zk_dump + @zk_wrapper = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new( + @admin.getConfiguration(), + "admin", + nil) + zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() + @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) end @@ -210,22 +225,33 @@ def create(table_name, *args) unless arg.kind_of?(String) || arg.kind_of?(Hash) raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") end - + # First, handle all the cases where arg is a column family. if arg.kind_of?(String) or arg.has_key?(NAME) # If the arg is a string, default action is to add a column to the table. # If arg has a name, it must also be a column descriptor. - htd.addFamily(hcd(arg, htd)) + descriptor = hcd(arg, htd); + # Warn if duplicate columns are added + if htd.hasFamily(descriptor.getName) + puts "Family '" + descriptor.getNameAsString() + "' already exists, the old one will be replaced" + htd.modifyFamily(descriptor) + else + htd.addFamily(descriptor) + end has_columns = true next end - + if arg.has_key?(REGION_REPLICATION) + region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION)) + htd.setRegionReplication(region_replication) + end + # Get rid of the "METHOD", which is deprecated for create. # We'll do whatever it used to do below if it's table_att. if (method = arg.delete(METHOD)) raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' end - + # The hash is not a column family. Figure out what's in it. # First, handle splits. if arg.has_key?(SPLITS_FILE) @@ -237,9 +263,10 @@ def create(table_name, *args) File.foreach(splits_file) do |line| arg[SPLITS].push(line.strip()) end + htd.setValue(SPLITS_FILE, arg[SPLITS_FILE]) end - if arg.has_key?(SPLITS) + if arg.has_key?(SPLITS) splits = Java::byte[][arg[SPLITS].size].new idx = 0 arg.delete(SPLITS).each do |split| @@ -255,26 +282,34 @@ def create(table_name, *args) split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) splits = split_algo.split(JInteger.valueOf(num_regions)) end - + # Done with splits; apply formerly-table_att parameters. - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] + # DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. + # However, it has to be set before DURABILITY so that DURABILITY could overwrite if both args are set + if arg.include?(DEFERRED_LOG_FLUSH) + if arg.delete(DEFERRED_LOG_FLUSH).to_s.upcase == "TRUE" + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("ASYNC_WAL")) + else + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("SYNC_WAL")) + end + end htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - + arg.each_key do |ignored_key| puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ]) end end - + # Fail if no column families defined raise(ArgumentError, "Table must have at least one column family") if !has_columns - + if splits.nil? # Perform the create table call @admin.createTable(htd) @@ -283,15 +318,15 @@ def create(table_name, *args) @admin.createTable(htd, splits) end end - + #---------------------------------------------------------------------------------------------- # Closes a region. # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) + if (server == nil || !closeEncodedRegion?(region_name, server)) @admin.closeRegion(region_name, server) - end + end end #---------------------------------------------------------------------------------------------- @@ -312,7 +347,7 @@ def unassign(region_name, force) def move(encoded_region_name, server = nil) @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) end - + #---------------------------------------------------------------------------------------------- # Merge two regions def merge_region(encoded_region_a_name, encoded_region_b_name, force) @@ -322,40 +357,77 @@ def merge_region(encoded_region_a_name, encoded_region_b_name, force) #---------------------------------------------------------------------------------------------- # Returns table's structure description def describe(table_name) - @admin.getTableDescriptor(table_name.to_java_bytes).to_s + @admin.getTableDescriptor(TableName.valueOf(table_name)).to_s + end + + def get_column_families(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).getColumnFamilies() + end + + def get_table_attributes(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).toStringTableAttributes end #---------------------------------------------------------------------------------------------- # Truncates table (deletes all records by recreating the table) def truncate(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - table_description = h_table.getTableDescriptor() - raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) + table_description = @admin.getTableDescriptor(TableName.valueOf(table_name)) + raise ArgumentError, "Table #{table_name} is not enabled. Enable it first." unless enabled?(table_name) yield 'Disabling table...' if block_given? @admin.disableTable(table_name) - yield 'Dropping table...' if block_given? - @admin.deleteTable(table_name) - - yield 'Creating table...' if block_given? - @admin.createTable(table_description) + begin + yield 'Truncating table...' if block_given? + @admin.truncateTable(org.apache.hadoop.hbase.TableName.valueOf(table_name), false) + rescue => e + # Handle the compatibility case, where the truncate method doesn't exists on the Master + raise e unless e.respond_to?(:cause) && e.cause != nil + rootCause = e.cause + if rootCause.kind_of?(org.apache.hadoop.hbase.DoNotRetryIOException) then + # Handle the compatibility case, where the truncate method doesn't exists on the Master + yield 'Dropping table...' if block_given? + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + + yield 'Creating table...' if block_given? + @admin.createTable(table_description) + else + raise e + end + end end #---------------------------------------------------------------------------------------------- # Truncates table while maintaing region boundaries (deletes all records by recreating the table) def truncate_preserve(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - splits = h_table.getRegionLocations().keys().map{|i| Bytes.toString(i.getStartKey)}.delete_if{|k| k == ""}.to_java :String - splits = org.apache.hadoop.hbase.util.Bytes.toByteArrays(splits) - table_description = h_table.getTableDescriptor() + h_table = @connection.getTable(TableName.valueOf(table_name)) + locator = @connection.getRegionLocator(TableName.valueOf(table_name)) + splits = locator.getAllRegionLocations(). + map{|i| Bytes.toString(i.getRegionInfo().getStartKey)}. + delete_if{|k| k == ""}.to_java :String + locator.close() + + table_description = @admin.getTableDescriptor(TableName.valueOf(table_name)) yield 'Disabling table...' if block_given? disable(table_name) - yield 'Dropping table...' if block_given? - drop(table_name) - - yield 'Creating table with region boundaries...' if block_given? - @admin.createTable(table_description, splits) + begin + yield 'Truncating table...' if block_given? + @admin.truncateTable(org.apache.hadoop.hbase.TableName.valueOf(table_name), true) + rescue => e + # Handle the compatibility case, where the truncate method doesn't exists on the Master + raise e unless e.respond_to?(:cause) && e.cause != nil + rootCause = e.cause + if rootCause.kind_of?(org.apache.hadoop.hbase.DoNotRetryIOException) then + # Handle the compatibility case, where the truncate method doesn't exists on the Master + yield 'Dropping table...' if block_given? + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + + yield 'Creating table with region boundaries...' if block_given? + @admin.createTable(table_description, splits) + else + raise e + end + end end #---------------------------------------------------------------------------------------------- @@ -393,18 +465,18 @@ def alter(table_name, wait = true, *args) raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? # Get table descriptor - htd = @admin.getTableDescriptor(table_name.to_java_bytes) + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) # Process all args args.each do |arg| - - + + # Normalize args to support column name only alter specs arg = { NAME => arg } if arg.kind_of?(String) # Normalize args to support shortcut delete syntax arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - + # There are 3 possible options. # 1) Column family spec. Distinguished by having a NAME and no METHOD. method = arg.delete(METHOD) @@ -423,12 +495,12 @@ def alter(table_name, wait = true, *args) puts "Updating all regions with the new schema..." alter_status(table_name) end - + # We bypass descriptor when adding column families; refresh it to apply other args correctly. - htd = @admin.getTableDescriptor(table_name.to_java_bytes) + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) next end - + # 2) Method other than table_att, with some args. name = arg.delete(NAME) if method != nil and method != "table_att" @@ -439,41 +511,59 @@ def alter(table_name, wait = true, *args) # Unset table attributes elsif method == "table_att_unset" raise(ArgumentError, "NAME parameter missing for table_att_unset method") unless name - if (htd.getValue(name) == nil) - raise ArgumentError, "Can not find attribute: #{name}" + if name.kind_of?(Array) + name.each do |key| + if (htd.getValue(key) == nil) + raise ArgumentError, "Could not find attribute: #{key}" + end + htd.remove(key) + end + else + if (htd.getValue(name) == nil) + raise ArgumentError, "Could not find attribute: #{name}" + end + htd.remove(name) end - htd.remove(name) @admin.modifyTable(table_name.to_java_bytes, htd) # Unknown method else raise ArgumentError, "Unknown method: #{method}" end - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) end - + if method == "delete" # We bypass descriptor when deleting column families; refresh it to apply other args correctly. - htd = @admin.getTableDescriptor(table_name.to_java_bytes) + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) end - next + next end - + # 3) Some args for the table, optionally with METHOD => table_att (deprecated) raise(ArgumentError, "NAME argument in an unexpected place") if name - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] + # DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. + # However, it has to be set before DURABILITY so that DURABILITY could overwrite if both args are set + if arg.include?(DEFERRED_LOG_FLUSH) + if arg.delete(DEFERRED_LOG_FLUSH).to_s.upcase == "TRUE" + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("ASYNC_WAL")) + else + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("SYNC_WAL")) + end + end htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] + htd.setRegionReplication(JInteger.valueOf(arg.delete(REGION_REPLICATION))) if arg[REGION_REPLICATION] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] @@ -485,39 +575,23 @@ def alter(table_name, wait = true, *args) k.strip! if (k =~ /coprocessor/i) - # validate coprocessor specs v = String.new(value) v.strip! - if !(v =~ /^([^\|]*)\|([^\|]+)\|[\s]*([\d]*)[\s]*(\|.*)?$/) - raise ArgumentError, "Coprocessor value doesn't match spec: #{v}" - end - - # generate a coprocessor ordinal by checking max id of existing cps - maxId = 0 - htd.getValues().each do |k1, v1| - attrName = org.apache.hadoop.hbase.util.Bytes.toString(k1.get()) - # a cp key is coprocessor$(\d) - if (attrName =~ /coprocessor\$(\d+)/i) - ids = attrName.scan(/coprocessor\$(\d+)/i) - maxId = ids[0][0].to_i if ids[0][0].to_i > maxId - end - end - maxId += 1 - htd.setValue(k + "\$" + maxId.to_s, value) + htd.addCoprocessor(v) valid_coproc_keys << key end end - + valid_coproc_keys.each do |key| arg.delete(key) end @admin.modifyTable(table_name.to_java_bytes, htd) - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) @@ -527,7 +601,7 @@ def alter(table_name, wait = true, *args) end end - def status(format) + def status(format, type) status = @admin.getClusterStatus() if format == "detailed" puts("version %s" % [ status.getHBaseVersion() ]) @@ -554,6 +628,46 @@ def status(format) for server in status.getDeadServerNames() puts(" %s" % [ server ]) end + elsif format == "replication" + #check whether replication is enabled or not + if (!@admin.getConfiguration().getBoolean(org.apache.hadoop.hbase.HConstants::REPLICATION_ENABLE_KEY, + org.apache.hadoop.hbase.HConstants::REPLICATION_ENABLE_DEFAULT)) + puts("Please enable replication first.") + else + puts("version %s" % [ status.getHBaseVersion() ]) + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + sl = status.getLoad(server) + rSinkString = " SINK :" + rSourceString = " SOURCE:" + rLoadSink = sl.getReplicationLoadSink() + rSinkString << " AgeOfLastAppliedOp=" + rLoadSink.getAgeOfLastAppliedOp().to_s + rSinkString << ", TimeStampsOfLastAppliedOp=" + + (java.util.Date.new(rLoadSink.getTimeStampsOfLastAppliedOp())).toString() + rLoadSourceList = sl.getReplicationLoadSourceList() + index = 0 + while index < rLoadSourceList.size() + rLoadSource = rLoadSourceList.get(index) + rSourceString << " PeerID=" + rLoadSource.getPeerID() + rSourceString << ", AgeOfLastShippedOp=" + rLoadSource.getAgeOfLastShippedOp().to_s + rSourceString << ", SizeOfLogQueue=" + rLoadSource.getSizeOfLogQueue().to_s + rSourceString << ", TimeStampsOfLastShippedOp=" + + (java.util.Date.new(rLoadSource.getTimeStampOfLastShippedOp())).toString() + rSourceString << ", Replication Lag=" + rLoadSource.getReplicationLag().to_s + index = index + 1 + end + puts(" %s:" % + [ server.getHostname() ]) + if type.casecmp("SOURCE") == 0 + puts("%s" % rSourceString) + elsif type.casecmp("SINK") == 0 + puts("%s" % rSinkString) + else + puts("%s" % rSourceString) + puts("%s" % rSinkString) + end + end + end elsif format == "simple" load = 0 regions = 0 @@ -584,7 +698,7 @@ def status(format) def exists?(table_name) @admin.tableExists(table_name) end - + #---------------------------------------------------------------------------------------------- # Is table enabled def enabled?(table_name) @@ -595,7 +709,7 @@ def enabled?(table_name) #Is supplied region name is encoded region name def closeEncodedRegion?(region_name, server) @admin.closeRegionWithEncodedRegionName(region_name, server) - end + end #---------------------------------------------------------------------------------------------- # Return a new HColumnDescriptor made of passed args @@ -611,19 +725,22 @@ def hcd(arg, htd) family.setBlockCacheEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) family.setScope(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) + family.setCacheDataOnWrite(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE) family.setInMemory(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) family.setTimeToLive(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::TTL))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) family.setBlocksize(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) family.setMaxVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) family.setMinVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + family.setKeepDeletedCells(org.apache.hadoop.hbase.KeepDeletedCells.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS).to_s.upcase)) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + family.setCompressTags(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS) + family.setPrefetchBlocksOnOpen(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN) family.setValue(COMPRESSION_COMPACT, arg.delete(COMPRESSION_COMPACT)) if arg.include?(COMPRESSION_COMPACT) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) + else family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) end end @@ -635,6 +752,16 @@ def hcd(arg, htd) family.setCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) end end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION) + algorithm = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION).upcase + family.setEncryptionType(algorithm) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY) + key = org.apache.hadoop.hbase.io.crypto.Encryption.pbkdf128( + arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY)) + family.setEncryptionKey(org.apache.hadoop.hbase.security.EncryptionUtil.wrapKey(@conf, key, + algorithm)) + end + end set_user_metadata(family, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(family, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] @@ -642,7 +769,7 @@ def hcd(arg, htd) arg.each_key do |unknown_key| puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key]) end - + return family end @@ -650,8 +777,7 @@ def hcd(arg, htd) # Enables/disables a region by name def online(region_name, on_off) # Open meta table - meta = org.apache.hadoop.hbase.client.HTable.new( - org.apache.hadoop.hbase.TableName::META_TABLE_NAME) + meta = @connection.getTable(org.apache.hadoop.hbase.TableName::META_TABLE_NAME) # Read region info # FIXME: fail gracefully if can't find the region @@ -677,11 +803,21 @@ def set_user_metadata(descriptor, metadata) descriptor.setValue(k, v) end end - + #---------------------------------------------------------------------------------------------- # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + def snapshot(table, snapshot_name, *args) + if args.empty? + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + else + args.each do |arg| + if arg[SKIP_FLUSH] == true + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH) + else + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + end + end end #---------------------------------------------------------------------------------------------- @@ -696,22 +832,22 @@ def clone_snapshot(snapshot_name, table) @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) end - #---------------------------------------------------------------------------------------------- - # Rename specified snapshot - def rename_snapshot(old_snapshot_name, new_snapshot_name) - @admin.renameSnapshot(old_snapshot_name.to_java_bytes, new_snapshot_name.to_java_bytes) - end - #---------------------------------------------------------------------------------------------- # Delete specified snapshot def delete_snapshot(snapshot_name) @admin.deleteSnapshot(snapshot_name.to_java_bytes) end + #---------------------------------------------------------------------------------------------- + # Deletes the snapshots matching the given regex + def delete_all_snapshot(regex) + @admin.deleteSnapshots(regex).to_a + end + #---------------------------------------------------------------------------------------------- # Returns a list of snapshots - def list_snapshot - @admin.listSnapshots + def list_snapshot(regex = ".*") + @admin.listSnapshots(regex).to_a end # Apply config specific to a table/column to its descriptor @@ -723,6 +859,18 @@ def set_descriptor_config(descriptor, config) end end + #---------------------------------------------------------------------------------------------- + # Updates the configuration of one regionserver. + def update_config(serverName) + @admin.updateConfiguration(ServerName.valueOf(serverName)); + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers. + def update_all_config() + @admin.updateConfiguration(); + end + #---------------------------------------------------------------------------------------------- # Returns namespace's structure description def describe_namespace(namespace_name) @@ -737,8 +885,10 @@ def describe_namespace(namespace_name) #---------------------------------------------------------------------------------------------- # Returns a list of namespaces in hbase - def list_namespace - @admin.listNamespaceDescriptors.map { |ns| ns.getName } + def list_namespace(regex = ".*") + pattern = java.util.regex.Pattern.compile(regex) + list = @admin.listNamespaceDescriptors.map { |ns| ns.getName } + list.select {|s| pattern.match(s) } end #---------------------------------------------------------------------------------------------- diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb similarity index 72% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb index 64482c5e0d..89700a486b 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb @@ -22,6 +22,7 @@ require 'hbase/admin' require 'hbase/table' require 'hbase/security' +require 'hbase/visibility_labels' module Hbase class Hbase @@ -35,17 +36,19 @@ def initialize(config = nil) self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. configuration.setInt("hbase.client.retries.number", 7) - configuration.setInt("ipc.client.connect.max.retries", 3) + configuration.setInt("hbase.ipc.client.connect.max.retries", 3) end + @connection = org.apache.hadoop.hbase.client.ConnectionFactory.createConnection( + self.configuration) end def admin(formatter) - ::Hbase::Admin.new(configuration, formatter) + ::Hbase::Admin.new(@connection.getAdmin, formatter) end # Create new one each time def table(table, shell) - ::Hbase::Table.new(configuration, table, shell) + ::Hbase::Table.new(@connection.getTable(table), shell) end def replication_admin(formatter) @@ -53,7 +56,15 @@ def replication_admin(formatter) end def security_admin(formatter) - ::Hbase::SecurityAdmin.new(configuration, formatter) + ::Hbase::SecurityAdmin.new(@connection.getAdmin, formatter) + end + + def visibility_labels_admin(formatter) + ::Hbase::VisibilityLabelsAdmin.new(@connection.getAdmin, formatter) + end + + def shutdown + @connection.close end end end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb new file mode 100644 index 0000000000..617073b6a0 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb @@ -0,0 +1,174 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +java_import org.apache.hadoop.hbase.client.replication.ReplicationAdmin +java_import org.apache.hadoop.hbase.replication.ReplicationPeerConfig +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.zookeeper.ZKUtil +java_import org.apache.hadoop.hbase.TableName + +# Wrapper for org.apache.hadoop.hbase.client.replication.ReplicationAdmin + +module Hbase + class RepAdmin + include HBaseConstants + + def initialize(configuration, formatter) + @replication_admin = ReplicationAdmin.new(configuration) + @configuration = configuration + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + # Add a new peer cluster to replicate to + def add_peer(id, args = {}, peer_tableCFs = nil) + # make add_peer backwards compatible to take in string for clusterKey and peer_tableCFs + if args.is_a?(String) + cluster_key = args + @replication_admin.addPeer(id, cluster_key, peer_tableCFs) + elsif args.is_a?(Hash) + unless peer_tableCFs.nil? + raise(ArgumentError, "peer_tableCFs should be specified as TABLE_CFS in args") + end + + endpoint_classname = args.fetch(ENDPOINT_CLASSNAME, nil) + cluster_key = args.fetch(CLUSTER_KEY, nil) + + # Handle cases where custom replication endpoint and cluster key are either both provided + # or neither are provided + if endpoint_classname.nil? and cluster_key.nil? + raise(ArgumentError, "Either ENDPOINT_CLASSNAME or CLUSTER_KEY must be specified.") + elsif !endpoint_classname.nil? and !cluster_key.nil? + raise(ArgumentError, "ENDPOINT_CLASSNAME and CLUSTER_KEY cannot both be specified.") + end + + # Cluster Key is required for ReplicationPeerConfig for a custom replication endpoint + if !endpoint_classname.nil? and cluster_key.nil? + cluster_key = ZKUtil.getZooKeeperClusterKey(@configuration) + end + + # Optional parameters + config = args.fetch(CONFIG, nil) + data = args.fetch(DATA, nil) + table_cfs = args.fetch(TABLE_CFS, nil) + + # Create and populate a ReplicationPeerConfig + replication_peer_config = ReplicationPeerConfig.new + replication_peer_config.set_cluster_key(cluster_key) + + unless endpoint_classname.nil? + replication_peer_config.set_replication_endpoint_impl(endpoint_classname) + end + + unless config.nil? + replication_peer_config.get_configuration.put_all(config) + end + + unless data.nil? + # Convert Strings to Bytes for peer_data + peer_data = replication_peer_config.get_peer_data + data.each{|key, val| + peer_data.put(Bytes.to_bytes(key), Bytes.to_bytes(val)) + } + end + + @replication_admin.add_peer(id, replication_peer_config, table_cfs) + else + raise(ArgumentError, "args must be either a String or Hash") + end + end + + #---------------------------------------------------------------------------------------------- + # Remove a peer cluster, stops the replication + def remove_peer(id) + @replication_admin.removePeer(id) + end + + + #--------------------------------------------------------------------------------------------- + # Show replcated tables/column families, and their ReplicationType + def list_replicated_tables(regex = ".*") + pattern = java.util.regex.Pattern.compile(regex) + list = @replication_admin.listReplicated() + list.select {|s| pattern.match(s.get(ReplicationAdmin::TNAME))} + end + + #---------------------------------------------------------------------------------------------- + # List all peer clusters + def list_peers + @replication_admin.listPeers + end + + #---------------------------------------------------------------------------------------------- + # Get peer cluster state + def get_peer_state(id) + @replication_admin.getPeerState(id) ? "ENABLED" : "DISABLED" + end + + #---------------------------------------------------------------------------------------------- + # Restart the replication stream to the specified peer + def enable_peer(id) + @replication_admin.enablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Stop the replication stream to the specified peer + def disable_peer(id) + @replication_admin.disablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Show the current tableCFs config for the specified peer + def show_peer_tableCFs(id) + @replication_admin.getPeerTableCFs(id) + end + + #---------------------------------------------------------------------------------------------- + # Set new tableCFs config for the specified peer + def set_peer_tableCFs(id, tableCFs) + @replication_admin.setPeerTableCFs(id, tableCFs) + end + + #---------------------------------------------------------------------------------------------- + # Append a tableCFs config for the specified peer + def append_peer_tableCFs(id, tableCFs) + @replication_admin.appendPeerTableCFs(id, tableCFs) + end + + #---------------------------------------------------------------------------------------------- + # Remove some tableCFs from the tableCFs config of the specified peer + def remove_peer_tableCFs(id, tableCFs) + @replication_admin.removePeerTableCFs(id, tableCFs) + end + #---------------------------------------------------------------------------------------------- + # Enables a table's replication switch + def enable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @replication_admin.enableTableRep(tableName) + end + #---------------------------------------------------------------------------------------------- + # Disables a table's replication switch + def disable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @replication_admin.disableTableRep(tableName) + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb similarity index 50% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb index e370ca9671..bfdd7330ff 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb @@ -24,12 +24,16 @@ module Hbase class SecurityAdmin include HBaseConstants - def initialize(configuration, formatter) - @config = configuration - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) + def initialize(admin, formatter) + @admin = admin + @config = @admin.getConfiguration() @formatter = formatter end + def close + @admin.close + end + #---------------------------------------------------------------------------------------------- def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) security_available? @@ -37,39 +41,31 @@ def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) # TODO: need to validate user name begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - perm = org.apache.hadoop.hbase.security.access.Permission.new( - permissions.to_java_bytes) - # Verify that the specified permission is valid if (permissions == nil || permissions.length == 0) raise(ArgumentError, "Invalid permission: no actions associated with user") end + perm = org.apache.hadoop.hbase.security.access.Permission.new( + permissions.to_java_bytes) + if (table_name != nil) tablebytes=table_name.to_java_bytes #check if the tablename passed is actually a namespace if (isNamespace?(table_name)) # Namespace should exist first. namespace_name = table_name[1...table_name.length] - raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) + raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless + namespace_exists?(namespace_name) - #We pass the namespace name along with "@" so that we can differentiate a namespace from a table. - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, tablebytes, perm.getActions()) + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, namespace_name, user, perm.getActions()) else # Table should exist raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name.to_java_bytes) - htd = @admin.getTableDescriptor(tablebytes) + htd = @admin.getTableDescriptor(tableName) if (family != nil) raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) @@ -78,19 +74,14 @@ def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) fambytes = family.to_java_bytes if (family != nil) qualbytes = qualifier.to_java_bytes if (qualifier != nil) - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, tableName, fambytes, - qualbytes, perm.getActions()) + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, tableName, user, fambytes, qualbytes, perm.getActions()) end else - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, perm.getActions()) + # invoke cp endpoint to perform access controls + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, user, perm.getActions()) end - - ensure - meta_table.close() end end @@ -101,14 +92,6 @@ def revoke(user, table_name=nil, family=nil, qualifier=nil) # TODO: need to validate user name begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - if (table_name != nil) #check if the tablename passed is actually a namespace if (isNamespace?(table_name)) @@ -116,11 +99,9 @@ def revoke(user, table_name=nil, family=nil, qualifier=nil) namespace_name = table_name[1...table_name.length] raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) - #We pass the namespace name along with "@" so that we can differentiate a namespace from a table. tablebytes=table_name.to_java_bytes - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke( - protocol, user, tablebytes) + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, namespace_name, user) else # Table should exist raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) @@ -135,71 +116,47 @@ def revoke(user, table_name=nil, family=nil, qualifier=nil) fambytes = family.to_java_bytes if (family != nil) qualbytes = qualifier.to_java_bytes if (qualifier != nil) - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke( - protocol, user, tableName, fambytes, qualbytes) + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, tableName, user, fambytes, qualbytes) end else - # invoke cp endpoint to perform access controlse perm = org.apache.hadoop.hbase.security.access.Permission.new(''.to_java_bytes) - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke(protocol, user, perm.getActions()) + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, user, perm.getActions()) end - ensure - meta_table.close() end end #---------------------------------------------------------------------------------------------- - def user_permission(table_name=nil) + def user_permission(table_regex=nil) security_available? - - begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - - if (table_name != nil) - #check if namespace is passed. - if (isNamespace?(table_name)) - # Namespace should exist first. - namespace_name = table_name[1...table_name.length] - raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) - # invoke cp endpoint to perform access controls - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions( - protocol, table_name.to_java_bytes) - else - raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name) - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions( - protocol, org.apache.hadoop.hbase.TableName.valueOf(table_name)) - end - else - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions(protocol) - end - ensure - meta_table.close() - end - + all_perms = org.apache.hadoop.hbase.security.access.AccessControlClient.getUserPermissions(@config,table_regex) res = {} count = 0 - perms.each do |value| - user_name = String.from_java_bytes(value.getUser) - table = (value.getTable != nil) ? value.getTable.toString() : '' - family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' - qualifier = (value.getQualifier != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : '' - - action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions - - if block_given? - yield(user_name, "#{table},#{family},#{qualifier}: #{action.to_s}") - else - res[user_name] ||= {} - res[user_name][family + ":" +qualifier] = action - end - count += 1 + all_perms.each do |value| + user_name = String.from_java_bytes(value.getUser) + if (table_regex != nil && isNamespace?(table_regex)) + namespace = table_regex[1...table_regex.length] + else + namespace = (value.getTableName != nil) ? value.getTableName.getNamespaceAsString() : '' + end + table = (value.getTableName != nil) ? value.getTableName.getNameAsString() : '' + family = (value.getFamily != nil) ? + org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : + '' + qualifier = (value.getQualifier != nil) ? + org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : + '' + + action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions + + if block_given? + yield(user_name, "#{namespace},#{table},#{family},#{qualifier}: #{action.to_s}") + else + res[user_name] ||= {} + res[user_name][family + ":" +qualifier] = action + end + count += 1 end return ((block_given?) ? count : res) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb similarity index 70% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb index 13d7646be3..e895302be8 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb @@ -19,7 +19,7 @@ include Java -# Wrapper for org.apache.hadoop.hbase.client.HTable +# Wrapper for org.apache.hadoop.hbase.client.Table module Hbase class Table @@ -49,7 +49,7 @@ def self.add_command(name, shell_command, internal_method_name) end end end - + # General help for the table # class level so we can call it from anywhere def self.help @@ -103,7 +103,7 @@ def self.help is undefined (and not recommended). EOF end - + #--------------------------------------------------------------------------------------------- # let external objects read the underlying table object @@ -111,25 +111,46 @@ def self.help # let external objects read the table name attr_reader :name - def initialize(configuration, table_name, shell) - if @@thread_pool then - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name.to_java_bytes, @@thread_pool) - else - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name) - @@thread_pool = @table.getPool() - end - @name = table_name + def initialize(table, shell) + @table = table + @name = @table.getName().getNameAsString() @shell = shell @converters = Hash.new() end + def close() + @table.close() + end + # Note the below methods are prefixed with '_' to hide them from the average user, as # they will be much less likely to tab complete to the 'dangerous' internal method #---------------------------------------------------------------------------------------------- + # Put a cell 'value' at specified table/row/column - def _put_internal(row, column, value, timestamp = nil) + def _put_internal(row, column, value, timestamp = nil, args = {}) p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) family, qualifier = parse_column_name(column) + if args.any? + attributes = args[ATTRIBUTES] + set_attributes(p, attributes) if attributes + visibility = args[VISIBILITY] + set_cell_visibility(p, visibility) if visibility + ttl = args[TTL] + set_op_ttl(p, ttl) if ttl + end + #Case where attributes are specified without timestamp + if timestamp.kind_of?(Hash) + timestamp.each do |k, v| + if k == 'ATTRIBUTES' + set_attributes(p, v) + elsif k == 'VISIBILITY' + set_cell_visibility(p, v) + elsif k == "TTL" + set_op_ttl(p, v) + end + end + timestamp = nil + end if timestamp p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) else @@ -140,15 +161,36 @@ def _put_internal(row, column, value, timestamp = nil) #---------------------------------------------------------------------------------------------- # Delete a cell - def _delete_internal(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - _deleteall_internal(row, column, timestamp) + def _delete_internal(row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + _deleteall_internal(row, column, timestamp, args) end #---------------------------------------------------------------------------------------------- # Delete a row - def _deleteall_internal(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - raise ArgumentError, "Row Not Found" if _get_internal(row).nil? + def _deleteall_internal(row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + # delete operation doesn't need read permission. Retaining the read check for + # meta table as a part of HBASE-5837. + if is_meta_table? + raise ArgumentError, "Row Not Found" if _get_internal(row).nil? + end + temptimestamp = timestamp + if temptimestamp.kind_of?(Hash) + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP + end d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp) + if temptimestamp.kind_of?(Hash) + temptimestamp.each do |k, v| + if v.kind_of?(String) + set_cell_visibility(d, v) if v + end + end + end + if args.any? + visibility = args[VISIBILITY] + set_cell_visibility(d, visibility) if visibility + end if column family, qualifier = parse_column_name(column) d.deleteColumns(family, qualifier, timestamp) @@ -158,13 +200,51 @@ def _deleteall_internal(row, column = nil, timestamp = org.apache.hadoop.hbase.H #---------------------------------------------------------------------------------------------- # Increment a counter atomically - def _incr_internal(row, column, value = nil) + def _incr_internal(row, column, value = nil, args={}) + if value.kind_of?(Hash) + value = 1 + end value ||= 1 + incr = org.apache.hadoop.hbase.client.Increment.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if qualifier.nil? + raise ArgumentError, "Failed to provide both column family and column qualifier for incr" + end + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(incr, attributes) if attributes + set_cell_visibility(incr, visibility) if visibility + ttl = args[TTL] + set_op_ttl(incr, ttl) if ttl + end + incr.addColumn(family, qualifier, value) + result = @table.increment(incr) + return nil if result.isEmpty + + # Fetch cell value + cell = result.listCells[0] + org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) + end + + #---------------------------------------------------------------------------------------------- + # appends the value atomically + def _append_internal(row, column, value, args={}) + append = org.apache.hadoop.hbase.client.Append.new(row.to_s.to_java_bytes) family, qualifier = parse_column_name(column) if qualifier.nil? - raise ArgumentError, "Failed to provide both column family and column qualifier for incr" + raise ArgumentError, "Failed to provide both column family and column qualifier for append" end - @table.incrementColumnValue(row.to_s.to_java_bytes, family, qualifier, value) + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(append, attributes) if attributes + set_cell_visibility(append, visibility) if visibility + ttl = args[TTL] + set_op_ttl(append, ttl) if ttl + end + append.add(family, qualifier, value.to_s.to_java_bytes) + @table.append(append) end #---------------------------------------------------------------------------------------------- @@ -172,8 +252,8 @@ def _incr_internal(row, column, value = nil) def _count_internal(interval = 1000, caching_rows = 10) # We can safely set scanner caching with the first key only filter scan = org.apache.hadoop.hbase.client.Scan.new - scan.cache_blocks = false - scan.caching = caching_rows + scan.setCacheBlocks(false) + scan.setCaching(caching_rows) scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) # Run the scanner @@ -187,10 +267,11 @@ def _count_internal(interval = 1000, caching_rows = 10) count += 1 next unless (block_given? && count % interval == 0) # Allow command modules to visualize counting process - yield(count, + yield(count, org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) end + scanner.close() # Return the counter return count end @@ -201,7 +282,7 @@ def _get_internal(row, *args) get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) maxlength = -1 @converters.clear() - + # Normalize args args = args.first if args.first.kind_of?(Hash) if args.kind_of?(String) || args.kind_of?(Array) @@ -219,7 +300,10 @@ def _get_internal(row, *args) # Get maxlength parameter if passed maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] filter = args.delete(FILTER) if args[FILTER] - + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args.delete(CONSISTENCY) if args[CONSISTENCY] + replicaId = args.delete(REGION_REPLICA_ID) if args[REGION_REPLICA_ID] unless args.empty? columns = args[COLUMN] || args[COLUMNS] if args[VERSIONS] @@ -251,9 +335,15 @@ def _get_internal(row, *args) get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] else - # May have passed TIMESTAMP and row only; wants all columns from ts. - unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] - raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + if attributes + set_attributes(get, attributes) + elsif authorizations + set_authorizations(get, authorizations) + else + # May have passed TIMESTAMP and row only; wants all columns from ts. + unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] + raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + end end get.setMaxVersions(vers) @@ -261,6 +351,8 @@ def _get_internal(row, *args) get.setTimeStamp(ts.to_i) if args[TIMESTAMP] get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] end + set_attributes(get, attributes) if attributes + set_authorizations(get, authorizations) if authorizations end unless filter.class == String @@ -269,6 +361,9 @@ def _get_internal(row, *args) get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) end + get.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency + get.setReplicaId(replicaId) if replicaId + # Call hbase for the results result = @table.get(get) return nil if result.isEmpty @@ -311,31 +406,26 @@ def _get_counter_internal(row, column) org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) end - #---------------------------------------------------------------------------------------------- - # Scans whole table or a range of keys and returns rows matching specific criteria - def _scan_internal(args = {}) - unless args.kind_of?(Hash) - raise ArgumentError, "Arguments should be a hash. Failed to parse #{args.inspect}, #{args.class}" - end - - limit = args.delete("LIMIT") || -1 - maxlength = args.delete("MAXLENGTH") || -1 - @converters.clear() - + def _hash_to_scan(args) if args.any? filter = args["FILTER"] startrow = args["STARTROW"] || '' stoprow = args["STOPROW"] timestamp = args["TIMESTAMP"] columns = args["COLUMNS"] || args["COLUMN"] || [] - cache_blocks = args["CACHE_BLOCKS"] || true + # If CACHE_BLOCKS not set, then default 'true'. + cache_blocks = args["CACHE_BLOCKS"].nil? ? true: args["CACHE_BLOCKS"] cache = args["CACHE"] || 0 + reversed = args["REVERSED"] || false versions = args["VERSIONS"] || 1 timerange = args[TIMERANGE] raw = args["RAW"] || false - + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args[CONSISTENCY] # Normalize column names columns = [columns] if columns.class == String + limit = args["LIMIT"] || -1 unless columns.kind_of?(Array) raise ArgumentError.new("COLUMNS must be specified as a String or an Array") end @@ -363,26 +453,44 @@ def _scan_internal(args = {}) scan.setTimeStamp(timestamp) if timestamp scan.setCacheBlocks(cache_blocks) + scan.setReversed(reversed) scan.setCaching(cache) if cache > 0 scan.setMaxVersions(versions) if versions > 1 scan.setTimeRange(timerange[0], timerange[1]) if timerange scan.setRaw(raw) + scan.setCaching(limit) if limit > 0 + set_attributes(scan, attributes) if attributes + set_authorizations(scan, authorizations) if authorizations + scan.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency else scan = org.apache.hadoop.hbase.client.Scan.new end - # Start the scanner - scanner = @table.getScanner(scan) + scan + end + + def _get_scanner(args) + @table.getScanner(_hash_to_scan(args)) + end + + #---------------------------------------------------------------------------------------------- + # Scans whole table or a range of keys and returns rows matching specific criteria + def _scan_internal(args = {}) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + + limit = args["LIMIT"] || -1 + maxlength = args.delete("MAXLENGTH") || -1 count = 0 res = {} + + @converters.clear() + + # Start the scanner + scanner = @table.getScanner(_hash_to_scan(args)) iter = scanner.iterator # Iterate results while iter.hasNext - if limit > 0 && count >= limit - break - end - row = iter.next key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) @@ -403,11 +511,53 @@ def _scan_internal(args = {}) # One more row processed count += 1 + if limit > 0 && count >= limit + # If we reached the limit, exit before the next call to hasNext + break + end end + scanner.close() return ((block_given?) ? count : res) end + # Apply OperationAttributes to puts/scans/gets + def set_attributes(oprattr, attributes) + raise(ArgumentError, "Attributes must be a Hash type") unless attributes.kind_of?(Hash) + for k,v in attributes + v = v.to_s unless v.nil? + oprattr.setAttribute(k.to_s, v.to_java_bytes) + end + end + + def set_cell_permissions(op, permissions) + raise(ArgumentError, "Permissions must be a Hash type") unless permissions.kind_of?(Hash) + map = java.util.HashMap.new + permissions.each do |user,perms| + map.put(user.to_s, org.apache.hadoop.hbase.security.access.Permission.new( + perms.to_java_bytes)) + end + op.setACL(map) + end + + def set_cell_visibility(oprattr, visibility) + oprattr.setCellVisibility( + org.apache.hadoop.hbase.security.visibility.CellVisibility.new( + visibility.to_s)) + end + + def set_authorizations(oprattr, authorizations) + raise(ArgumentError, "Authorizations must be a Array type") unless authorizations.kind_of?(Array) + auths = [ authorizations ].flatten.compact + oprattr.setAuthorizations( + org.apache.hadoop.hbase.security.visibility.Authorizations.new( + auths.to_java(:string))) + end + + def set_op_ttl(op, ttl) + op.setTTL(ttl.to_java(:long)) + end + #---------------------------- # Add general administration utilities to the shell # each of the names below adds this method name to the table @@ -465,9 +615,7 @@ def get_all_columns # Checks if current table is one of the 'meta' tables def is_meta_table? - tn = @table.table_name - org.apache.hadoop.hbase.util.Bytes.equals(tn, - org.apache.hadoop.hbase.TableName::META_TABLE_NAME.getName) + org.apache.hadoop.hbase.TableName::META_TABLE_NAME.equals(@table.getName()) end # Returns family and (when has it) qualifier for a column name @@ -502,7 +650,7 @@ def to_string(column, kv, maxlength = -1) end (maxlength != -1) ? val[0, maxlength] : val end - + def convert(column, kv) #use org.apache.hadoop.hbase.util.Bytes as the default class klazz_name = 'org.apache.hadoop.hbase.util.Bytes' @@ -514,7 +662,7 @@ def convert(column, kv) if matches.nil? # cannot match the pattern of 'c(className).functionname' # use the default klazz_name - converter = @converters[column] + converter = @converters[column] else klazz_name = matches[1] converter = matches[2] @@ -523,7 +671,7 @@ def convert(column, kv) method = eval(klazz_name).method(converter) return method.call(kv.getValue) # apply the converter end - + # if the column spec contains CONVERTER information, to get rid of :CONVERTER info from column pair. # 1. return back normal column pair as usual, i.e., "cf:qualifier[:CONVERTER]" to "cf" and "qualifier" only # 2. register the CONVERTER information based on column spec - "cf:qualifier" diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb new file mode 100644 index 0000000000..4b99bf9026 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb @@ -0,0 +1,148 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.security.visibility.VisibilityClient +java_import org.apache.hadoop.hbase.security.visibility.VisibilityConstants +java_import org.apache.hadoop.hbase.util.Bytes + +module Hbase + class VisibilityLabelsAdmin + + def initialize(admin, formatter) + @admin = admin + @config = @admin.getConfiguration() + @formatter = formatter + end + + def close + @admin.close + end + + def add_labels(*args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + labels = [ args ].flatten.compact + end + if labels.size() == 0 + raise(ArgumentError, "Arguments cannot be null") + end + + begin + response = VisibilityClient.addLabels(@config, labels.to_java(:string)) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + def set_auths(user, *args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + auths = [ args ].flatten.compact + end + + begin + response = VisibilityClient.setAuths(@config, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + def get_auths(user) + lables_table_available? + begin + response = VisibilityClient.getAuths(@config, user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + return response.getAuthList + end + end + + def list_labels(regex = ".*") + lables_table_available? + begin + response = VisibilityClient.listLabels(@config, regex) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + return response.getLabelList + end + end + + def clear_auths(user, *args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + auths = [ args ].flatten.compact + end + + begin + response = VisibilityClient.clearAuths(@config, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + # Make sure that lables table is available + def lables_table_available?() + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") \ + unless exists?(VisibilityConstants::LABELS_TABLE_NAME) + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb similarity index 87% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb index 7c1ed17812..994a195d27 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb @@ -29,7 +29,7 @@ def self.command_groups @@command_groups end - def self.load_command(name, group) + def self.load_command(name, group, aliases=[]) return if commands[name] # Register command in the group @@ -41,6 +41,9 @@ def self.load_command(name, group) require "shell/commands/#{name}" klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize commands[name] = eval("Commands::#{klass_name}") + aliases.each do |an_alias| + commands[an_alias] = commands[name] + end rescue => e raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" end @@ -56,8 +59,11 @@ def self.load_command_group(group, opts) :comment => opts[:comment] } + all_aliases = opts[:aliases] || {} + opts[:commands].each do |command| - load_command(command, group) + aliases = all_aliases[command] || [] + load_command(command, group, aliases) end end @@ -90,6 +96,10 @@ def hbase_security_admin @hbase_security_admin ||= hbase.security_admin(formatter) end + def hbase_visibility_labels_admin + @hbase_visibility_labels_admin ||= hbase.visibility_labels_admin(formatter) + end + def export_commands(where) ::Shell.commands.keys.each do |cmd| # here where is the IRB namespace @@ -208,7 +218,7 @@ def help_footer hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. -For more on the HBase Shell, see http://hbase.apache.org/docs/current/book.html +For more on the HBase Shell, see http://hbase.apache.org/book.html HERE end end @@ -250,7 +260,10 @@ def help_footer alter_status alter_async get_table - ] + ], + :aliases => { + 'describe' => ['desc'] + } ) Shell.load_command_group( @@ -280,6 +293,7 @@ def help_footer scan truncate truncate_preserve + append ] ) @@ -300,12 +314,17 @@ def help_footer merge_region unassign zk_dump - hlog_roll + wal_roll catalogjanitor_run catalogjanitor_switch catalogjanitor_enabled + compact_rs trace - ] + ], + # TODO remove older hlog_roll command + :aliases => { + 'wal_roll' => ['hlog_roll'] + } ) Shell.load_command_group( @@ -318,23 +337,38 @@ def help_footer list_peers enable_peer disable_peer + show_peer_tableCFs + set_peer_tableCFs list_replicated_tables + append_peer_tableCFs + remove_peer_tableCFs + enable_table_replication + disable_table_replication ] ) Shell.load_command_group( - 'snapshot', + 'snapshots', :full_name => 'CLUSTER SNAPSHOT TOOLS', :commands => %w[ snapshot clone_snapshot restore_snapshot - rename_snapshot delete_snapshot + delete_all_snapshot list_snapshots ] ) +Shell.load_command_group( + 'configuration', + :full_name => 'ONLINE CONFIGURATION TOOLS', + :commands => %w[ + update_config + update_all_config + ] +) + Shell.load_command_group( 'security', :full_name => 'SECURITY TOOLS', @@ -346,3 +380,16 @@ def help_footer ] ) +Shell.load_command_group( + 'visibility labels', + :full_name => 'VISIBILITY LABEL TOOLS', + :comment => "NOTE: Above commands are only applicable if running with the VisibilityController coprocessor", + :commands => %w[ + add_labels + list_labels + set_auths + get_auths + clear_auths + set_visibility + ] +) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb similarity index 78% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb index 4303aa6e45..54fa204048 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb @@ -62,6 +62,10 @@ def security_admin @shell.hbase_security_admin end + def visibility_labels_admin + @shell.hbase_visibility_labels_admin + end + #---------------------------------------------------------------------- def formatter @@ -87,18 +91,32 @@ def translate_hbase_exceptions(*args) yield rescue => e raise e unless e.respond_to?(:cause) && e.cause != nil - + # Get the special java exception which will be handled cause = e.cause if cause.kind_of?(org.apache.hadoop.hbase.TableNotFoundException) then - raise "Unknown table #{args.first}!" + str = java.lang.String.new("#{cause}") + raise "Unknown table #{str}!" end - if cause.kind_of?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) then - valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } - raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" + if cause.kind_of?(org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException) then + exceptions = cause.getCauses + exceptions.each do |exception| + if exception.kind_of?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) then + valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } + raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" + end + end end if cause.kind_of?(org.apache.hadoop.hbase.TableExistsException) then - raise "Table already exists: #{args.first}!" + str = java.lang.String.new("#{cause}") + strs = str.split("\n") + if strs.size > 0 then + s = strs[0].split(' '); + if(s.size > 1) + raise "Table already exists: #{s[1]}!" + end + raise "Table already exists: #{strs[0]}!" + end end # To be safe, here only AccessDeniedException is considered. In future # we might support more in more generic approach when possible. @@ -112,7 +130,7 @@ def translate_hbase_exceptions(*args) end end - # Throw the other exception which hasn't been handled above + # Throw the other exception which hasn't been handled above raise e end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb similarity index 80% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb index b8c3791a54..65a1140f24 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb @@ -18,18 +18,21 @@ module Shell module Commands - class DeleteSnapshot < Command + class AddLabels < Command def help return <<-EOF -Delete a specified snapshot. Examples: +Add a set of visibility labels. +Syntax : add_labels [label1, label2] - hbase> delete_snapshot 'snapshotName', +For example: + + hbase> add_labels ['SECRET','PRIVATE'] EOF end - def command(snapshot_name) + def command(*args) format_simple_command do - admin.delete_snapshot(snapshot_name) + visibility_labels_admin.add_labels(args) end end end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb new file mode 100644 index 0000000000..be01041644 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb @@ -0,0 +1,70 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddPeer< Command + def help + return <<-EOF +A peer can either be another HBase cluster or a custom replication endpoint. In either case an id +must be specified to identify the peer. + +For a HBase cluster peer, a cluster key must be provided and is composed like this: +hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent +This gives a full path for HBase to connect to another HBase cluster. An optional parameter for +table column families identifies which column families will be replicated to the peer cluster. +Examples: + + hbase> add_peer '1', "server1.cie.com:2181:/hbase" + hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" + hbase> add_peer '3', "zk4,zk5,zk6:11000:/hbase-test", "table1; table2:cf1; table3:cf1,cf2" + hbase> add_peer '4', CLUSTER_KEY => "server1.cie.com:2181:/hbase" + hbase> add_peer '5', CLUSTER_KEY => "server1.cie.com:2181:/hbase", + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + +For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments +are DATA and CONFIG which can be specified to set different either the peer_data or configuration +for the custom replication endpoint. Table column families is optional and can be specified with +the key TABLE_CFS. + + hbase> add_peer '6', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint' + hbase> add_peer '7', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 } + hbase> add_peer '8', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + CONFIG => { "config1" => "value1", "config2" => "value2" } + hbase> add_peer '9', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + hbase> add_peer '10', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + hbase> add_peer '11', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + +Note: Either CLUSTER_KEY or ENDPOINT_CLASSNAME must be specified but not both. +EOF + end + + def command(id, args = {}, peer_tableCFs = nil) + format_simple_command do + replication_admin.add_peer(id, args, peer_tableCFs) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb similarity index 72% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb index f22348ce44..2c3aa6f872 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb @@ -22,12 +22,17 @@ module Commands class Alter < Command def help return <<-EOF -Alter a table. Depending on the HBase setting ("hbase.online.schema.update.enable"), -the table must be disabled or not to be altered (see help 'disable'). -You can add/modify/delete column families, as well as change table -configuration. Column families work similarly to create; column family -spec can either be a name string, or a dictionary with NAME attribute. -Dictionaries are described on the main help command output. +Alter a table. If the "hbase.online.schema.update.enable" property is set to +false, then the table must be disabled (see help 'disable'). If the +"hbase.online.schema.update.enable" property is set to true, tables can be +altered without disabling them first. Altering enabled tables has caused problems +in the past, so use caution and test it before using in production. + +You can use the alter command to add, +modify or delete column families or change table configuration options. +Column families work in a similar way as the 'create' command. The column family +specification can either be a name string, or a dictionary with the NAME attribute. +Dictionaries are described in the output of the 'help' command, with no arguments. For example, to change or add the 'f1' column family in table 't1' from current value to keep a maximum of 5 cell VERSIONS, do: @@ -38,13 +43,13 @@ def help hbase> alter 't1', 'f1', {NAME => 'f2', IN_MEMORY => true}, {NAME => 'f3', VERSIONS => 5} -To delete the 'f1' column family in table 't1', use one of: +To delete the 'f1' column family in table 'ns1:t1', use one of: - hbase> alter 't1', NAME => 'f1', METHOD => 'delete' - hbase> alter 't1', 'delete' => 'f1' + hbase> alter 'ns1:t1', NAME => 'f1', METHOD => 'delete' + hbase> alter 'ns1:t1', 'delete' => 'f1' You can also change table-scope attributes like MAX_FILESIZE, READONLY, -MEMSTORE_FLUSHSIZE, DEFERRED_LOG_FLUSH, etc. These can be put at the end; +MEMSTORE_FLUSHSIZE, DURABILITY, etc. These can be put at the end; for example, to change the max size of a region to 128MB, do: hbase> alter 't1', MAX_FILESIZE => '134217728' @@ -74,6 +79,10 @@ def help hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' +You can also set REGION_REPLICATION: + + hbase> alter 't1', {REGION_REPLICATION => 2} + There could be more than one alteration in one command: hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb index 164f339149..bddff010a2 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb @@ -32,13 +32,13 @@ def help hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 -To delete the 'f1' column family in table 't1', do: +To delete the 'f1' column family in table 'ns1:t1', do: - hbase> alter_async 't1', NAME => 'f1', METHOD => 'delete' + hbase> alter_async 'ns1:t1', NAME => 'f1', METHOD => 'delete' or a shorter version: - hbase> alter_async 't1', 'delete' => 'f1' + hbase> alter_async 'ns1:t1', 'delete' => 'f1' You can also change table-scope attributes like MAX_FILESIZE MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb index d62edacc9c..72f044dbba 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb @@ -27,6 +27,7 @@ def help Pass table name. hbase> alter_status 't1' +hbase> alter_status 'ns1:t1' EOF end def command(table) diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb new file mode 100644 index 0000000000..a0ef36d28e --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb @@ -0,0 +1,52 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Append < Command + def help + return <<-EOF +Appends a cell 'value' at specified table/row/column coordinates. + + hbase> append 't1', 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> append 't1', 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.append 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> t.append 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, value, args={}) + append(table(table), row, column, value, args) + end + + def append(table, row, column, value, args={}) + format_simple_command do + table._append_internal(row, column, value, args) + end + end + end + end +end + +#add incr comamnd to Table +::Hbase::Table.add_shell_command("append") diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb similarity index 75% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb index ba597662c5..3919b20317 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,20 +19,21 @@ module Shell module Commands - class Flush < Command + class AppendPeerTableCFs< Command def help return <<-EOF -Flush all regions in passed table or pass a region row to -flush an individual region. For example: +Append a replicable table-cf config for the specified peer +Examples: + + # append a table / table-cf to be replicable for a peer + hbase> append_peer_tableCFs '2', "table4:cfA,cfB" - hbase> flush 'TABLENAME' - hbase> flush 'REGIONNAME' EOF end - def command(table_or_region_name) + def command(id, table_cfs) format_simple_command do - admin.flush(table_or_region_name) + replication_admin.append_peer_tableCFs(id, table_cfs) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb new file mode 100644 index 0000000000..8553fa6471 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearAuths < Command + def help + return <<-EOF +Clear visibility labels from a user or group +Syntax : clear_auths 'user',[label1, label2] + +For example: + + hbase> clear_auths 'user1', ['SECRET','PRIVATE'] + hbase> clear_auths '@group1', ['SECRET','PRIVATE'] +EOF + end + + def command(user, *args) + format_simple_command do + visibility_labels_admin.clear_auths(user, args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb similarity index 95% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb index c4d5b0b2d5..8c193bb7c1 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb @@ -27,6 +27,7 @@ def help Examples: hbase> clone_snapshot 'snapshotName', 'tableName' + hbase> clone_snapshot 'snapshotName', 'namespace:tableName' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb similarity index 96% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb index b7cdf3dca2..9e2900c31b 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb @@ -30,6 +30,8 @@ def help name looks like this: TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. +or + Namespace:TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. The trailing period is part of the regionserver name. A region's encoded name is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb index 47cf0380ac..adeffa2d03 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb @@ -27,6 +27,7 @@ def help family within a region. Examples: Compact all regions in a table: + hbase> compact 'ns1:t1' hbase> compact 't1' Compact an entire region: hbase> compact 'r1' diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb similarity index 63% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb index 7669fb7c52..0ecdd21113 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,23 +19,23 @@ module Shell module Commands - class AddPeer< Command + class CompactRs < Command def help return <<-EOF -Add a peer cluster to replicate to, the id must be a short and -the cluster key is composed like this: -hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent -This gives a full path for HBase to connect to another cluster. -Examples: - - hbase> add_peer '1', "server1.cie.com:2181:/hbase" - hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" -EOF + Compact all regions on passed regionserver. + Examples: + Compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020' + or + hbase> compact_rs 'host187.example.com,60020,1289493121758' + Major compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020,1289493121758', true + EOF end - def command(id, cluster_key) + def command(regionserver, major = false) format_simple_command do - replication_admin.add_peer(id, cluster_key) + admin.compact_regionserver(regionserver, major) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb similarity index 99% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb index e88c445d5b..225005ee36 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb @@ -30,6 +30,7 @@ def help If your rows are small in size, you may want to increase this parameter. Examples: + hbase> count 'ns1:t1' hbase> count 't1' hbase> count 't1', INTERVAL => 100000 hbase> count 't1', CACHE => 1000 diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb similarity index 87% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb index e59a5cb65d..ab3a3d1069 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb @@ -29,7 +29,10 @@ def help including NAME attribute. Examples: - hbase> create 't1', {NAME => 'f1', VERSIONS => 5} +Create a table with namespace=ns1 and table qualifier=t1 + hbase> create 'ns1:t1', {NAME => 'f1', VERSIONS => 5} + +Create a table with namespace=default and table qualifier=t1 hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} hbase> # The above in shorthand would be the following: hbase> create 't1', 'f1', 'f2', 'f3' @@ -39,13 +42,14 @@ def help Table configuration options can be put at the end. Examples: + hbase> create 'ns1:t1', 'f1', SPLITS => ['10', '20', '30', '40'] hbase> create 't1', 'f1', SPLITS => ['10', '20', '30', '40'] hbase> create 't1', 'f1', SPLITS_FILE => 'splits.txt', OWNER => 'johndoe' hbase> create 't1', {NAME => 'f1', VERSIONS => 5}, METADATA => { 'mykey' => 'myvalue' } hbase> # Optionally pre-split the table into NUMREGIONS, using hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} - hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'}} + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit', REGION_REPLICATION => 2, CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'}} You can also keep around a reference to the created table: diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb similarity index 74% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb index b0bca71277..dcb83414f6 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb @@ -28,22 +28,27 @@ def help versions. To delete a cell from 't1' at row 'r1' under column 'c1' marked with the time 'ts1', do: + hbase> delete 'ns1:t1', 'r1', 'c1', ts1 hbase> delete 't1', 'r1', 'c1', ts1 + hbase> delete 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} The same command can also be run on a table reference. Suppose you had a reference t to table 't1', the corresponding command would be: hbase> t.delete 'r1', 'c1', ts1 + hbase> t.delete 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} EOF end - def command(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - delete(table(table), row, column, timestamp) + def command(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + delete(table(table), row, column, timestamp, args) end - def delete(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + def delete(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) format_simple_command do - table._delete_internal(row, column, timestamp) + table._delete_internal(row, column, timestamp, args) end end end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb new file mode 100644 index 0000000000..bc072597bb --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteAllSnapshot < Command + def help + return <<-EOF +Delete all of the snapshots matching the given regex. Examples: + + hbase> delete_all_snapshot 's.*' + +EOF + end + + def command(regex) + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + list = admin.list_snapshot(regex) + count = list.size + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + puts "\nDelete the above #{count} snapshots (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No snapshots matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + format_simple_command do + admin.delete_all_snapshot(regex) + end + list = admin.list_snapshot(regex) + leftOverSnapshotCount = list.size + successfullyDeleted = count - leftOverSnapshotCount + puts "#{successfullyDeleted} snapshots successfully deleted." unless successfullyDeleted == 0 + return if leftOverSnapshotCount == 0 + puts "\nFailed to delete the below #{leftOverSnapshotCount} snapshots." + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb similarity index 82% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb index a09a882927..e6118c9144 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb @@ -25,9 +25,11 @@ def help Delete all cells in a given row; pass a table name, row, and optionally a column and timestamp. Examples: + hbase> deleteall 'ns1:t1', 'r1' hbase> deleteall 't1', 'r1' hbase> deleteall 't1', 'r1', 'c1' hbase> deleteall 't1', 'r1', 'c1', ts1 + hbase> deleteall 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} The same commands also can be run on a table reference. Suppose you had a reference t to table 't1', the corresponding command would be: @@ -35,18 +37,19 @@ def help hbase> t.deleteall 'r1' hbase> t.deleteall 'r1', 'c1' hbase> t.deleteall 'r1', 'c1', ts1 + hbase> t.deleteall 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} EOF end def command(table, row, column = nil, - timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - deleteall(table(table), row, column, timestamp) + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + deleteall(table(table), row, column, timestamp, args) end def deleteall(table, row, column = nil, - timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) format_simple_command do - table._deleteall_internal(row, column, timestamp) + table._deleteall_internal(row, column, timestamp, args) end end end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb similarity index 65% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb index 0f355074cd..bfa16cdc2d 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -25,16 +24,25 @@ def help return <<-EOF Describe the named table. For example: hbase> describe 't1' + hbase> describe 'ns1:t1' + +Alternatively, you can use the abbreviated 'desc' for the same thing. + hbase> desc 't1' + hbase> desc 'ns1:t1' EOF end def command(table) now = Time.now - desc = admin.describe(table) + column_families = admin.get_column_families(table) - formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) - formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) + formatter.header(["Table " + table.to_s + " is " + if admin.enabled?(table) then "ENABLED" else "DISABLED" end]) + formatter.row([table.to_s + admin.get_table_attributes(table)], true) + formatter.header(["COLUMN FAMILIES DESCRIPTION"]) + column_families.each do |column_family| + formatter.row([ column_family.to_s ], true) + end formatter.footer(now) end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb index b10aa9bf03..79bcd864f7 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb @@ -22,7 +22,9 @@ module Commands class Disable < Command def help return <<-EOF -Start disable of named table: e.g. "hbase> disable 't1'" +Start disable of named table: + hbase> disable 't1' + hbase> disable 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb similarity index 94% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb index 0e7c30ee99..212db249a8 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb @@ -25,12 +25,13 @@ def help Disable all of tables matching the given regex: hbase> disable_all 't.*' +hbase> disable_all 'ns:t.*' +hbase> disable_all 'ns:.*' EOF end def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) + list = admin.list(regex) count = list.size list.each do |table| formatter.row([ table ]) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb similarity index 76% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb index de459092b5..5bf9667949 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb @@ -1,6 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -20,22 +18,22 @@ module Shell module Commands - class EnablePeer< Command + class DisableTableReplication< Command def help return <<-EOF -Restarts the replication to the specified peer cluster, -continuing from where it was disabled. +Disable a table's replication switch. Examples: - hbase> enable_peer '1' + hbase> disable_table_replication 'table_name' EOF end - def command(id) + def command(table_name) format_simple_command do - replication_admin.enable_peer(id) + replication_admin.disable_tablerep(table_name) end + puts "The replication swith of table '#{table_name}' successfully disabled" end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb similarity index 91% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb index 6238fe65b8..fc7b134468 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb @@ -22,7 +22,9 @@ module Commands class Drop < Command def help return <<-EOF -Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'" +Drop the named table. Table must first be disabled: + hbase> drop 't1' + hbase> drop 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb similarity index 94% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb index e482d6b0c7..73398fbf6e 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb @@ -25,12 +25,13 @@ def help Drop all of the tables matching the given regex: hbase> drop_all 't.*' +hbase> drop_all 'ns:t.*' +hbase> drop_all 'ns:.*' EOF end def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) + list = admin.list(regex) count = list.size list.each do |table| formatter.row([ table ]) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb similarity index 93% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb index a4d495a55e..deeb70cce2 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb @@ -22,7 +22,9 @@ module Commands class Enable < Command def help return <<-EOF -Start enable of named table: e.g. "hbase> enable 't1'" +Start enable of named table: + hbase> enable 't1' + hbase> enable 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb similarity index 94% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb index fc0f081f15..2b899f2ac0 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb @@ -25,12 +25,13 @@ def help Enable all of the tables matching the given regex: hbase> enable_all 't.*' +hbase> enable_all 'ns:t.*' +hbase> enable_all 'ns:.*' EOF end def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) + list = admin.list(regex) count = list.size list.each do |table| formatter.row([ table ]) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb similarity index 76% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb index da9941a957..15e3133a89 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb @@ -1,6 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -20,22 +18,22 @@ module Shell module Commands - class DisablePeer< Command + class EnableTableReplication< Command def help return <<-EOF -Stops the replication stream to the specified cluster, but still -keeps track of new edits to replicate. +Enable a table's replication switch. Examples: - hbase> disable_peer '1' + hbase> enable_table_replication 'table_name' EOF end - def command(id) + def command(table_name) format_simple_command do - replication_admin.disable_peer(id) + replication_admin.enable_tablerep(table_name) end + puts "The replication swith of table '#{table_name}' successfully enabled" end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb similarity index 93% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb index b7932ced52..bacf6c94c4 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb @@ -22,7 +22,9 @@ module Commands class Exists < Command def help return <<-EOF -Does the named table exist? e.g. "hbase> exists 't1'" +Does the named table exist? + hbase> exists 't1' + hbase> exists 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb similarity index 86% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb index 4344d68d14..1ab13cb9ca 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb @@ -25,6 +25,7 @@ def help Get row or cell contents; pass table name, row, and optionally a dictionary of column(s), timestamp, timerange and versions. Examples: + hbase> get 'ns1:t1', 'r1' hbase> get 't1', 'r1' hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} hbase> get 't1', 'r1', {COLUMN => 'c1'} @@ -36,6 +37,10 @@ def help hbase> get 't1', 'r1', 'c1' hbase> get 't1', 'r1', 'c1', 'c2' hbase> get 't1', 'r1', ['c1', 'c2'] + hbase> get 't1', 'r1', {COLUMN => 'c1', ATTRIBUTES => {'mykey'=>'myvalue'}} + hbase> get 't1', 'r1', {COLUMN => 'c1', AUTHORIZATIONS => ['PRIVATE','SECRET']} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} Besides the default 'toStringBinary' format, 'get' also supports custom formatting by column. A user can define a FORMATTER by adding it to the column name in the get @@ -48,7 +53,7 @@ def help hbase> get 't1', 'r1' {COLUMN => ['cf:qualifier1:toInt', 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } -Note that you can specify a FORMATTER by column only (cf:qualifer). You cannot specify +Note that you can specify a FORMATTER by column only (cf:qualifier). You cannot specify a FORMATTER for all columns of a column family. The same commands also can be run on a reference to a table (obtained via get_table or @@ -66,6 +71,8 @@ def help hbase> t.get 'r1', 'c1' hbase> t.get 'r1', 'c1', 'c2' hbase> t.get 'r1', ['c1', 'c2'] + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} EOF end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb similarity index 70% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb index 721519e7d6..1b758ef949 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -18,21 +17,25 @@ module Shell module Commands - class Revoke < Command + class GetAuths < Command def help return <<-EOF -Revoke a user's access rights. -Syntax : revoke [
[ []] +Get the visibility labels set for a particular user or group +Syntax : get_auths 'user' + For example: - hbase> revoke 'bobsmith' - hbase> revoke 'bobsmith', 't1', 'f1', 'col1' + hbase> get_auths 'user1' + hbase> get_auths '@group1' EOF end - def command(user, table_name=nil, family=nil, qualifier=nil) + def command(user) format_simple_command do - security_admin.revoke(user, table_name, family, qualifier) + list = visibility_labels_admin.get_auths(user) + list.each do |auths| + formatter.row([org.apache.hadoop.hbase.util.Bytes::toStringBinary(auths.toByteArray)]) + end end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb similarity index 81% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb index ee71ae4978..1264fe5f0f 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb @@ -23,9 +23,10 @@ class GetCounter < Command def help return <<-EOF Return a counter cell value at specified table/row/column coordinates. -A cell cell should be managed with atomic increment function oh HBase -and the data should be binary encoded. Example: +A counter cell should be managed with atomic increment functions on HBase +and the data should be binary encoded (as long value). Example: + hbase> get_counter 'ns1:t1', 'r1', 'c1' hbase> get_counter 't1', 'r1', 'c1' The same commands also can be run on a table reference. Suppose you had a reference @@ -35,11 +36,11 @@ def help EOF end - def command(table, row, column, value) - get_counter(table(table), row, column, value) + def command(table, row, column, dummy = nil) + get_counter(table(table), row, column) end - def get_counter(table, row, column, value = nil) + def get_counter(table, row, column, dummy = nil) if cnt = table._get_counter_internal(row, column) puts "COUNTER VALUE = #{cnt}" else diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb index 76f4b952d7..43e7c1af70 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb @@ -27,6 +27,7 @@ def help Eg. hbase> t1 = get_table 't1' + hbase> t1 = get_table 'ns1:t1' returns the table named 't1' as a table object. You can then do diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb new file mode 100644 index 0000000000..4a995ffe52 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Grant < Command + def help + return <<-EOF +Grant users specific rights. +Syntax : grant [<@namespace> [
[ []]] + +permissions is either zero or more letters from the set "RWXCA". +READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') + +Note: Groups and users are granted access in the same way, but groups are prefixed with an '@' + character. In the same way, tables and namespaces are specified, but namespaces are + prefixed with an '@' character. + +For example: + + hbase> grant 'bobsmith', 'RWXCA' + hbase> grant '@admins', 'RWXCA' + hbase> grant 'bobsmith', 'RWXCA', '@ns1' + hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' + hbase> grant 'bobsmith', 'RW', 'ns1:t1', 'f1', 'col1' +EOF + end + + def command(*args) + + # command form is ambiguous at first argument + table_name = user = args[0] + raise(ArgumentError, "First argument should be a String") unless user.kind_of?(String) + + if args[1].kind_of?(String) + + # Original form of the command + # user in args[0] + # permissions in args[1] + # table_name in args[2] + # family in args[3] or nil + # qualifier in args[4] or nil + + permissions = args[1] + raise(ArgumentError, "Permissions are not of String type") unless permissions.kind_of?( + String) + table_name = family = qualifier = nil + table_name = args[2] # will be nil if unset + if not table_name.nil? + raise(ArgumentError, "Table name is not of String type") unless table_name.kind_of?( + String) + family = args[3] # will be nil if unset + if not family.nil? + raise(ArgumentError, "Family is not of String type") unless family.kind_of?(String) + qualifier = args[4] # will be nil if unset + if not qualifier.nil? + raise(ArgumentError, "Qualifier is not of String type") unless qualifier.kind_of?( + String) + end + end + end + format_simple_command do + security_admin.grant(user, permissions, table_name, family, qualifier) + end + + elsif args[1].kind_of?(Hash) + + # New form of the command, a cell ACL update + # table_name in args[0], a string + # a Hash mapping users (or groups) to permisisons in args[1] + # a Hash argument suitable for passing to Table#_get_scanner in args[2] + # Useful for feature testing and debugging. + + permissions = args[1] + raise(ArgumentError, "Permissions are not of Hash type") unless permissions.kind_of?(Hash) + scan = args[2] + raise(ArgumentError, "Scanner specification is not a Hash") unless scan.kind_of?(Hash) + + t = table(table_name) + now = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.list.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_permissions(put, permissions) + t.table.put(put) + end + count += 1 + end + formatter.footer(now, count) + + else + raise(ArgumentError, "Second argument should be a String or Hash") + end + + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb similarity index 62% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb index 6ebac29ace..d223a45aaf 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb @@ -23,29 +23,39 @@ class Incr < Command def help return <<-EOF Increments a cell 'value' at specified table/row/column coordinates. -To increment a cell value in table 't1' at row 'r1' under column +To increment a cell value in table 'ns1:t1' or 't1' at row 'r1' under column 'c1' by 1 (can be omitted) or 10 do: + hbase> incr 'ns1:t1', 'r1', 'c1' hbase> incr 't1', 'r1', 'c1' hbase> incr 't1', 'r1', 'c1', 1 hbase> incr 't1', 'r1', 'c1', 10 + hbase> incr 't1', 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} The same commands also can be run on a table reference. Suppose you had a reference t to table 't1', the corresponding command would be: hbase> t.incr 'r1', 'c1' hbase> t.incr 'r1', 'c1', 1 - hbase> t.incr 'r1', 'c1', 10 + hbase> t.incr 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> t.incr 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} EOF end - def command(table, row, column, value) - incr(table(table), row, column, value) + def command(table, row, column, value = nil, args = {}) + incr(table(table), row, column, value, args) end - def incr(table, row, column, value = nil) - cnt = table._incr_internal(row, column, value) - puts "COUNTER VALUE = #{cnt}" + def incr(table, row, column, value = nil, args={}) + format_simple_command do + if cnt = table._incr_internal(row, column, value, args) + puts "COUNTER VALUE = #{cnt}" + else + puts "No counter found at specified coordinates" + end + end end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb index 28918ee429..6da7046ba7 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb @@ -22,7 +22,9 @@ module Commands class IsDisabled < Command def help return <<-EOF -Is named table disabled?: e.g. "hbase> is_disabled 't1'" +Is named table disabled? For example: + hbase> is_disabled 't1' + hbase> is_disabled 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb index 6a98a36b71..960ade7572 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb @@ -22,7 +22,9 @@ module Commands class IsEnabled < Command def help return <<-EOF -Is named table enabled?: e.g. "hbase> is_enabled 't1'" +Is named table enabled? For example: + hbase> is_enabled 't1' + hbase> is_enabled 'ns1:t1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb similarity index 96% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb index 7801f530b9..dce0ae2acd 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb @@ -27,6 +27,8 @@ def help hbase> list hbase> list 'abc.*' + hbase> list 'ns:abc.*' + hbase> list 'ns:.*' EOF end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb new file mode 100644 index 0000000000..6c7f99137b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListLabels < Command + def help + return <<-EOF +List the visibility labels defined in the system. +Optional regular expression parameter could be used to filter the labels being returned. +Syntax : list_labels + +For example: + + hbase> list_labels 'secret.*' + hbase> list_labels +EOF + end + + def command(regex = ".*") + format_simple_command do + list = visibility_labels_admin.list_labels(regex) + list.each do |label| + formatter.row([org.apache.hadoop.hbase.util.Bytes::toStringBinary(label.toByteArray)]) + end + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb index 0b577fe2bd..5d25604afa 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb @@ -34,8 +34,7 @@ def command(regex = ".*") now = Time.now formatter.header([ "NAMESPACE" ]) - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_namespace.grep(regex) + list = admin.list_namespace(regex) list.each do |table| formatter.row([ table ]) end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb similarity index 86% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb index 2f57592e58..cc1be04416 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb @@ -33,11 +33,12 @@ def command() now = Time.now peers = replication_admin.list_peers - formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"]) + formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE", "TABLE_CFS"]) peers.entrySet().each do |e| state = replication_admin.get_peer_state(e.key) - formatter.row([ e.key, e.value, state ]) + tableCFs = replication_admin.show_peer_tableCFs(e.key) + formatter.row([ e.key, e.value, state, tableCFs ]) end formatter.footer(now) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb similarity index 88% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb index b1494b8d3a..0db1d83aeb 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb @@ -34,9 +34,7 @@ def command(regex = ".*") now = Time.now formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ]) - list = replication_admin.list_replicated_tables - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))} + list = replication_admin.list_replicated_tables(regex) list.each do |e| if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL replicateType = "GLOBAL" diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb similarity index 92% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb index 95135185c2..4e68802f8d 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb @@ -37,8 +37,7 @@ def command(regex = ".*") now = Time.now formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_snapshot.select {|s| regex.match(s.getName)} + list = admin.list_snapshot(regex) list.each do |snapshot| creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb index f05cc1d78d..825748336c 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb @@ -29,6 +29,7 @@ def help Examples: Compact all regions in a table: hbase> major_compact 't1' + hbase> major_compact 'ns1:t1' Compact an entire region: hbase> major_compact 'r1' Compact a single column family within a region: diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb similarity index 62% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb index 17e7a5f261..2b47a4d5c6 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb @@ -23,25 +23,30 @@ class Put < Command def help return <<-EOF Put a cell 'value' at specified table/row/column and optionally -timestamp coordinates. To put a cell value into table 't1' at -row 'r1' under column 'c1' marked with the time 'ts1', do: +timestamp coordinates. To put a cell value into table 'ns1:t1' or 't1' +at row 'r1' under column 'c1' marked with the time 'ts1', do: + hbase> put 'ns1:t1', 'r1', 'c1', 'value' + hbase> put 't1', 'r1', 'c1', 'value' hbase> put 't1', 'r1', 'c1', 'value', ts1 + hbase> put 't1', 'r1', 'c1', 'value', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {VISIBILITY=>'PRIVATE|SECRET'} The same commands also can be run on a table reference. Suppose you had a reference t to table 't1', the corresponding command would be: - hbase> t.put 'r1', 'c1', 'value', ts1 + hbase> t.put 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} EOF end - def command(table, row, column, value, timestamp = nil) - put table(table), row, column, value, timestamp + def command(table, row, column, value, timestamp=nil, args = {}) + put table(table), row, column, value, timestamp, args end - def put(table, row, column, value, timestamp = nil) + def put(table, row, column, value, timestamp = nil, args = {}) format_simple_command do - table._put_internal(row, column, value, timestamp) + table._put_internal(row, column, value, timestamp, args) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb similarity index 71% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb index 0eac7654a8..5b15b52965 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,22 +19,22 @@ module Shell module Commands - class BalanceSwitch < Command + class RemovePeerTableCFs < Command def help return <<-EOF -Enable/Disable balancer. Returns previous balancer state. +Remove a table / table-cf from the table-cfs config for the specified peer Examples: - hbase> balance_switch true - hbase> balance_switch false + # Remove a table / table-cf from the replicable table-cfs for a peer + hbase> remove_peer_tableCFs '2', "table1" + hbase> remove_peer_tableCFs '2', "table1:cf1" + EOF end - def command(enableDisable) + def command(id, table_cfs) format_simple_command do - formatter.row([ - admin.balance_switch(enableDisable)? "true" : "false" - ]) + replication_admin.remove_peer_tableCFs(id, table_cfs) end end end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb similarity index 74% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb index 721519e7d6..768989b6c0 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb @@ -22,11 +22,19 @@ class Revoke < Command def help return <<-EOF Revoke a user's access rights. -Syntax : revoke [
[ []] +Syntax : revoke [<@namespace> [
[ []]]] + +Note: Groups and users access are revoked in the same way, but groups are prefixed with an '@' + character. In the same way, tables and namespaces are specified, but namespaces are + prefixed with an '@' character. + For example: hbase> revoke 'bobsmith' + hbase> revoke '@admins' + hbase> revoke 'bobsmith', '@ns1' hbase> revoke 'bobsmith', 't1', 'f1', 'col1' + hbase> revoke 'bobsmith', 'ns1:t1', 'f1', 'col1' EOF end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb similarity index 88% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb index 364fd6e0b1..7e28b20b12 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb @@ -25,7 +25,7 @@ def help Scan a table; pass table name and optionally a dictionary of scanner specifications. Scanner specifications may include one or more of: TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, -or COLUMNS, CACHE +or COLUMNS, CACHE or RAW, VERSIONS If no columns are specified, all columns will be scanned. To scan all members of a column family, leave the qualifier empty as in @@ -40,13 +40,18 @@ def help hbase> scan 'hbase:meta' hbase> scan 'hbase:meta', {COLUMNS => 'info:regioninfo'} + hbase> scan 'ns1:t1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} + hbase> scan 't1', {REVERSED => true} hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} hbase> scan 't1', {FILTER => org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} - + hbase> scan 't1', {CONSISTENCY => 'TIMELINE'} +For setting the Operation Attributes + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], ATTRIBUTES => {'mykey' => 'myvalue'}} + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], AUTHORIZATIONS => ['PRIVATE','SECRET']} For experts, there is an additional option -- CACHE_BLOCKS -- which switches block caching for the scanner on (true) or off (false). By default it is enabled. Examples: @@ -71,7 +76,7 @@ def help hbase> scan 't1', {COLUMNS => ['cf:qualifier1:toInt', 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } -Note that you can specify a FORMATTER by column only (cf:qualifer). You cannot +Note that you can specify a FORMATTER by column only (cf:qualifier). You cannot specify a FORMATTER for all columns of a column family. Scan can also be used directly from a table, by first getting a reference to a diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb similarity index 74% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb index 034434aa5d..4a52eb0ef7 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb @@ -1,6 +1,3 @@ -# -# Copyright 2010 The Apache Software Foundation -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -20,19 +17,22 @@ module Shell module Commands - class RemovePeer< Command + class SetAuths < Command def help return <<-EOF -Stops the specified replication stream and deletes all the meta -information kept about it. Examples: +Add a set of visibility labels for a user or group +Syntax : set_auths 'user',[label1, label2] + +For example: - hbase> remove_peer '1' + hbase> set_auths 'user1', ['SECRET','PRIVATE'] + hbase> set_auths '@group1', ['SECRET','PRIVATE'] EOF end - def command(id) + def command(user, *args) format_simple_command do - replication_admin.remove_peer(id) + visibility_labels_admin.set_auths(user, args) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb similarity index 59% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb index 2f57592e58..3a88dbb741 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb @@ -20,27 +20,27 @@ module Shell module Commands - class ListPeers< Command + class SetPeerTableCFs< Command def help return <<-EOF -List all replication peer clusters. + Set the replicable table-cf config for the specified peer + Examples: - hbase> list_peers -EOF - end - - def command() - now = Time.now - peers = replication_admin.list_peers + # set all tables to be replicable for a peer + hbase> set_peer_tableCFs '1', "" + hbase> set_peer_tableCFs '1' + # set table / table-cf to be replicable for a peer, for a table without + # an explicit column-family list, all replicable column-families (with + # replication_scope == 1) will be replicated + hbase> set_peer_tableCFs '2', "table1; table2:cf1,cf2; table3:cfA,cfB" - formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"]) + EOF + end - peers.entrySet().each do |e| - state = replication_admin.get_peer_state(e.key) - formatter.row([ e.key, e.value, state ]) + def command(id, peer_table_cfs = nil) + format_simple_command do + replication_admin.set_peer_tableCFs(id, peer_table_cfs) end - - formatter.footer(now) end end end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb new file mode 100644 index 0000000000..b2b57b19c3 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetVisibility < Command + def help + return <<-EOF +Set the visibility expression on one or more existing cells. + +Pass table name, visibility expression, and a dictionary containing +scanner specifications. Scanner specifications may include one or more +of: TIMERANGE, FILTER, STARTROW, STOPROW, TIMESTAMP, or COLUMNS + +If no columns are specified, all columns will be included. +To include all members of a column family, leave the qualifier empty as in +'col_family:'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +Examples: + + hbase> set_visibility 't1', 'A|B', {COLUMNS => ['c1', 'c2']} + hbase> set_visibility 't1', '(A&B)|C', {COLUMNS => 'c1', + TIMERANGE => [1303668804, 1303668904]} + hbase> set_visibility 't1', 'A&B&C', {FILTER => "(PrefixFilter ('row2') AND + (QualifierFilter (>=, 'binary:xyz'))) AND + (TimestampsFilter ( 123, 456))"} + +This command will only affect existing cells and is expected to be mainly +useful for feature testing and functional verification. +EOF + end + + def command(table, visibility, scan) + t = table(table) + now = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.list.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_visibility(put, visibility) + t.table.put(put) + end + count += 1 + end + formatter.footer(now, count) + end + + end + end +end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb similarity index 79% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb index 716859fc06..cdbd9ed9b5 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb @@ -27,7 +27,6 @@ def help Show all the filters in hbase. Example: hbase> show_filters - Documentation on filters mentioned below can be found at: https://our.intern.facebook.com/intern/wiki/index.php/HBase/Filter_Language ColumnPrefixFilter TimestampsFilter PageFilter @@ -38,10 +37,6 @@ def help def command( ) now = Time.now - formatter.row(["Documentation on filters mentioned below can " + - "be found at: https://our.intern.facebook.com/intern/" + - "wiki/index.php/HBase/Filter_Language"]) - parseFilter = ParseFilter.new supportedFilters = parseFilter.getSupportedFilters diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb similarity index 76% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb index 40831d2fa0..3ce3d06f01 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb @@ -1,5 +1,5 @@ # -# Copyright 2010 The Apache Software Foundation +# Copyright The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,17 +20,17 @@ module Shell module Commands - class Drop < Command + class ShowPeerTableCFs< Command def help - return <<-EOF -Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'" -EOF + return <<-EOF + Show replicable table-cf config for the specified peer. + + hbase> show_peer_tableCFs + EOF end - def command(table) - format_simple_command do - admin.drop(table) - end + def command(id) + puts replication_admin.show_peer_tableCFs(id) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb similarity index 85% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb index 1c4ecfec0e..15bf298e04 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb @@ -24,12 +24,13 @@ def help Take a snapshot of specified table. Examples: hbase> snapshot 'sourceTable', 'snapshotName' + hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true} EOF end - def command(table, snapshot_name) + def command(table, snapshot_name, *args) format_simple_command do - admin.snapshot(table, snapshot_name) + admin.snapshot(table, snapshot_name, *args) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb index 9488ed9ef1..9dc424ff68 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb @@ -26,6 +26,7 @@ def help second parameter, you can specify an explicit split key for the region. Examples: split 'tableName' + split 'namespace:tableName' split 'regionName' # format: 'tableName,startKey,id' split 'tableName', 'splitKey' split 'regionName', 'splitKey' diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb similarity index 79% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb index 4b22acb4ab..b22b272398 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -23,18 +22,21 @@ module Commands class Status < Command def help return <<-EOF -Show cluster status. Can be 'summary', 'simple', or 'detailed'. The +Show cluster status. Can be 'summary', 'simple', 'detailed', or 'replication'. The default is 'summary'. Examples: hbase> status hbase> status 'simple' hbase> status 'summary' hbase> status 'detailed' + hbase> status 'replication' + hbase> status 'replication', 'source' + hbase> status 'replication', 'sink' EOF end - def command(format = 'summary') - admin.status(format) + def command(format = 'summary',type = 'both') + admin.status(format, type) end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb similarity index 96% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb index ba061957c2..5e0093092f 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb @@ -16,8 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -HTrace = org.cloudera.htrace.Trace -java_import org.cloudera.htrace.Sampler +HTrace = org.apache.htrace.Trace +java_import org.apache.htrace.Sampler java_import org.apache.hadoop.hbase.trace.SpanReceiverHost module Shell diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb similarity index 75% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb index 4c83d3cf20..05295b7996 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,18 +19,19 @@ module Shell module Commands - class Assign < Command + class UpdateAllConfig < Command def help return <<-EOF -Assign a region.Use with caution.If region already assigned, -this command will just go ahead and reassign -the region anyways. For experts only. +Reload a subset of configuration on all servers in the cluster. See +http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_all_config EOF end - def command(region_name) + def command() format_simple_command do - admin.assign(region_name) + admin.update_all_config() end end end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb similarity index 69% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb index 05ecd3a2bc..9f65fddd25 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb @@ -1,5 +1,4 @@ # -# Copyright 2010 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,20 +19,20 @@ module Shell module Commands - class Balancer < Command + class UpdateConfig < Command def help return <<-EOF -Trigger the cluster balancer. Returns true if balancer ran and was able to -tell the region servers to unassign all the regions to balance (the re-assignment itself is async). -Otherwise false (Will not run if regions in transition). +Reload a subset of configuration on server 'servername' where servername is +host, port plus startcode. For example: host187.example.com,60020,1289493121758 +See http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_config 'servername' EOF end - def command() + def command(serverName) format_simple_command do - formatter.row([ - admin.balancer()? "true": "false" - ]) + admin.update_config(serverName) end end end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb similarity index 73% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb index ad4a7b029e..e4673fc391 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb @@ -23,20 +23,27 @@ def help return <<-EOF Show all permissions for the particular user. Syntax : user_permission
+ +Note: A namespace must always precede with '@' character. + For example: hbase> user_permission + hbase> user_permission '@ns1' hbase> user_permission 'table1' + hbase> user_permission 'namespace1:table1' + hbase> user_permission '.*' + hbase> user_permission '^[A-C].*' EOF end - def command(table=nil) + def command(table_regex=nil) #format_simple_command do - #admin.user_permission(table) + #admin.user_permission(table_regex) now = Time.now - formatter.header(["User", "Table,Family,Qualifier:Permission"]) + formatter.header(["User", "Namespace,Table,Family,Qualifier:Permission"]) - count = security_admin.user_permission(table) do |user, permission| + count = security_admin.user_permission(table_regex) do |user, permission| formatter.row([ user, permission]) end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb similarity index 90% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb index 02c7c59282..0fe1870af2 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb @@ -1,5 +1,4 @@ # -# Copyright 2011 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -19,7 +18,7 @@ # module Shell module Commands - class HlogRoll < Command + class WalRoll < Command def help return <<-EOF Roll the log writer. That is, start writing log messages to a new file. @@ -32,9 +31,12 @@ def help def command(server_name) format_simple_command do - admin.hlog_roll(server_name) + admin.wal_roll(server_name) end end end + #TODO remove old HLog version + class HlogRoll < WalRoll + end end end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb similarity index 80% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb index 040ad7e7ef..3b6b32d257 100644 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb @@ -30,7 +30,12 @@ def help end def command() - puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}" + user = org.apache.hadoop.hbase.security.User.getCurrent() + puts "#{user.toString()}" + groups = user.getGroupNames().to_a + if not groups.nil? and groups.length > 0 + puts " groups: #{groups.join(', ')}" + end end end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb new file mode 100755 index 0000000000..9287e5bfc2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script is used to issue a stop command to a regionserver via RPC. +# Intended for use in environments where sshing around is inappropriate +# Run it like this by passing it to a jruby interpreter: +# +# ./bin/hbase org.jruby.Main bin/shutdown_regionserver.rb c2021:16020 + +include Java +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin + +def usage(msg=nil) + $stderr.puts 'Usage: shutdown_regionserver.rb ..' + $stderr.puts + $stderr.puts 'Stops the specified regionservers via RPC' + $stderr.puts 'Error: %s' % msg if msg + abort +end + +usage if ARGV.length < 1 + +ARGV.each do |x| + usage 'Invalid host:port: %s' % x unless x.include? ':' +end + +config = HBaseConfiguration.create() +begin + admin = HBaseAdmin.new(config) +rescue + abort "Error: Couldn't instantiate HBaseAdmin" +end + +ARGV.each do |hostport| + admin.stopRegionServer(hostport) +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd new file mode 100755 index 0000000000..676a11e6e2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd @@ -0,0 +1,61 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Modelled after $HADOOP_HOME/bin/start-hbase.sh. + +@rem Start hadoop hbase daemons. +@rem Run this on master node. +@echo off +set usage="Usage: start-hbase.cmd" + +setlocal + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call %hbase-config-script% + +set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed +for /f %%i in ('%distModeCommand%') do set distMode=%%i + +if "%distMode%"=="false" ( + start "HBase Distribution" %HBASE_BIN_PATH%\hbase.cmd master start +) else ( + if "%distMode%"=="true" ( + @echo This is not implemented yet. Stay tuned. + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "${HBASE_CONF_DIR}" start zookeeper + @rem call %HBASE_BIN_PATH%\hbase-daemon.cmd --config "${HBASE_CONF_DIR}" start master + + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_REGIONSERVERS%" start regionserver + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_BACKUP_MASTERS%" start master-backup + ) else ( + echo ERROR: Could not determine the startup mode. + ) +) + +@rem -------------- End of main script -------------- +endlocal +goto :eof \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh similarity index 94% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh index 3b096f01a3..aeb9469fb7 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information @@ -52,7 +50,7 @@ distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBa if [ "$distMode" == 'false' ] then - "$bin"/hbase-daemon.sh $commandToRun master $@ + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master $@ else "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" $commandToRun zookeeper "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd new file mode 100755 index 0000000000..97180554b5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd @@ -0,0 +1,54 @@ +@echo off +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Modelled after $HADOOP_HOME/bin/stop-hbase.sh. + +@rem Stop hadoop hbase daemons. Run this on master node. + +setlocal + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call %hbase-config-script% + +set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed +for /f %%i in ('%distModeCommand%') do set distMode=%%i + +if "%distMode%"=="false" ( + call %HBASE_BIN_PATH%\hbase.cmd master stop + +) else ( + if "%distMode%"=="true" ( + @echo This is not implemented yet. Stay tuned. + ) else ( + echo ERROR: Could not determine the startup mode. + ) +) + +@rem -------------- End of main script -------------- +endlocal +goto :eof \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh similarity index 97% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh index a9ad3460c6..8b81deedd8 100755 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2007 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh similarity index 100% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb old mode 100644 new mode 100755 similarity index 54% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb rename to titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb index 7bdc95943b..68140124c8 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,30 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. -module Shell - module Commands - class Grant < Command - def help - return <<-EOF -Grant users specific rights. -Syntax : grant [
[ []] - -permissions is either zero or more letters from the set "RWXCA". -READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') - -For example: - - hbase> grant 'bobsmith', 'RWXCA' - hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' -EOF - end +require 'thread' - def command(user, rights, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.grant(user, rights, table_name, family, qualifier) +class ThreadPool + def initialize(poolsize) + @queue = Queue.new + @poolsize = poolsize + @pool = Array.new(@poolsize) do |i| + Thread.new do + Thread.current[:id] = i + catch(:close) do + loop do + job, args = @queue.pop + job.call(*args) + end end end end end + + def launch(*args, &block) + @queue << [block, args] + end + + def stop + @poolsize.times do + launch { throw :close } + end + @pool.map(&:join) + end end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch b/titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch similarity index 73% rename from titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch rename to titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch index e17adde94e..afdd6d097c 100644 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch @@ -1,13 +1,11 @@ ---- /home/dalaro/hbase-0.96.1.1-hadoop1/bin/hbase 2013-12-17 14:05:37.000000000 -0500 -+++ /home/dalaro/tinkerelius/titan/titan-hbase/titan-hbase-096/src/test/bin/hbase 2014-02-12 11:25:18.000000000 -0500 -@@ -143,10 +143,24 @@ +--- /home/dalaro/hbase-1.0.2/bin/hbase 2015-08-25 18:34:24.000000000 -0400 ++++ hbase 2015-09-02 15:37:46.000000000 -0400 +@@ -168,6 +168,31 @@ + } + - add_maven_deps_to_classpath() { - f="${HBASE_HOME}/target/cached_classpath.txt" -- if [ ! -f "${f}" ] -- then -- echo "As this is a development environment, we need ${f} to be generated from maven (command: mvn install -DskipTests)" -- exit 1 ++add_maven_deps_to_classpath() { ++ f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ]; then + # ------------------- + # Titan build support @@ -26,10 +24,15 @@ + fi + mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` - } -@@ -262,12 +276,8 @@ ++ fi ++ CLASSPATH=${CLASSPATH}:`cat "${f}"` ++} ++ ++ + #Add the development env class path stuff + if $in_dev_env; then + add_maven_deps_to_classpath +@@ -286,13 +311,8 @@ CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" fi @@ -39,6 +42,7 @@ - else - HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/hbase-shell/src/main/ruby" - fi +- HBASE_OPTS="$HBASE_OPTS $HBASE_SHELL_OPTS" + # find the hbase ruby sources + HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh similarity index 97% rename from titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh rename to titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh index 89a214e5a8..97bf41b605 100755 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash # #/** -# * Copyright 2009 The Apache Software Foundation -# * # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information