From 9e711597a42582c2bc8030d0a2d306fdedf7b18b Mon Sep 17 00:00:00 2001 From: David Robinson Date: Mon, 14 Dec 2015 09:47:57 -0500 Subject: [PATCH 1/3] adjusting hbase support --- pom.xml | 5 +- titan-hbase-parent/pom.xml | 2 - titan-hbase-parent/titan-hbase-094/.gitignore | 1 - titan-hbase-parent/titan-hbase-094/pom.xml | 352 -------- .../google/common/io/NullOutputStream.java | 39 - .../diskstorage/hbase/HBaseAdmin0_94.java | 150 ---- .../diskstorage/hbase/HBaseCompat0_94.java | 42 - .../diskstorage/hbase/HConnection0_94.java | 35 - .../titan/diskstorage/hbase/HTable0_94.java | 46 - .../src/test/bin/get-active-master.rb | 45 - .../src/test/bin/graceful_stop.sh | 110 --- .../titan-hbase-094/src/test/bin/hbase | 373 -------- .../src/test/bin/hbase-config.sh | 120 --- .../src/test/bin/hbase-daemon.sh | 205 ----- .../src/test/bin/hbase-daemons.sh | 55 -- .../titan-hbase-094/src/test/bin/hbase-jruby | 22 - .../titan-hbase-094/src/test/bin/hirb.rb | 185 ---- .../src/test/bin/local-master-backup.sh | 55 -- .../src/test/bin/local-regionservers.sh | 54 -- .../src/test/bin/master-backup.sh | 76 -- .../src/test/bin/region_mover.rb | 475 ---------- .../src/test/bin/region_status.rb | 149 ---- .../src/test/bin/regionservers.sh | 75 -- .../src/test/bin/rolling-restart.sh | 165 ---- .../src/test/bin/ruby/hbase.rb | 80 -- .../src/test/bin/ruby/hbase/admin.rb | 667 -------------- .../src/test/bin/ruby/hbase/hbase.rb | 60 -- .../test/bin/ruby/hbase/replication_admin.rb | 88 -- .../src/test/bin/ruby/hbase/security.rb | 188 ---- .../src/test/bin/ruby/hbase/table.rb | 366 -------- .../src/test/bin/ruby/irb/hirb.rb | 60 -- .../src/test/bin/ruby/shell.rb | 315 ------- .../src/test/bin/ruby/shell/commands.rb | 85 -- .../test/bin/ruby/shell/commands/add_peer.rb | 44 - .../src/test/bin/ruby/shell/commands/alter.rb | 90 -- .../bin/ruby/shell/commands/alter_async.rb | 66 -- .../bin/ruby/shell/commands/alter_status.rb | 38 - .../test/bin/ruby/shell/commands/assign.rb | 39 - .../bin/ruby/shell/commands/balance_switch.rb | 43 - .../test/bin/ruby/shell/commands/balancer.rb | 41 - .../bin/ruby/shell/commands/clone_snapshot.rb | 40 - .../bin/ruby/shell/commands/close_region.rb | 58 -- .../test/bin/ruby/shell/commands/compact.rb | 48 - .../src/test/bin/ruby/shell/commands/count.rb | 61 -- .../test/bin/ruby/shell/commands/create.rb | 51 -- .../test/bin/ruby/shell/commands/delete.rb | 43 - .../ruby/shell/commands/delete_snapshot.rb | 37 - .../test/bin/ruby/shell/commands/deleteall.rb | 42 - .../test/bin/ruby/shell/commands/describe.rb | 42 - .../test/bin/ruby/shell/commands/disable.rb | 37 - .../bin/ruby/shell/commands/disable_all.rb | 50 -- .../bin/ruby/shell/commands/disable_peer.rb | 42 - .../src/test/bin/ruby/shell/commands/drop.rb | 37 - .../test/bin/ruby/shell/commands/drop_all.rb | 50 -- .../test/bin/ruby/shell/commands/enable.rb | 37 - .../bin/ruby/shell/commands/enable_all.rb | 50 -- .../bin/ruby/shell/commands/enable_peer.rb | 42 - .../test/bin/ruby/shell/commands/exists.rb | 39 - .../src/test/bin/ruby/shell/commands/flush.rb | 41 - .../src/test/bin/ruby/shell/commands/get.rb | 55 -- .../bin/ruby/shell/commands/get_counter.rb | 43 - .../src/test/bin/ruby/shell/commands/grant.rb | 44 - .../test/bin/ruby/shell/commands/hlog_roll.rb | 40 - .../src/test/bin/ruby/shell/commands/incr.rb | 42 - .../bin/ruby/shell/commands/is_disabled.rb | 39 - .../bin/ruby/shell/commands/is_enabled.rb | 39 - .../src/test/bin/ruby/shell/commands/list.rb | 48 - .../bin/ruby/shell/commands/list_peers.rb | 47 - .../shell/commands/list_replicated_tables.rb | 52 -- .../bin/ruby/shell/commands/list_snapshots.rb | 52 -- .../bin/ruby/shell/commands/major_compact.rb | 49 -- .../src/test/bin/ruby/shell/commands/move.rb | 48 - .../src/test/bin/ruby/shell/commands/put.rb | 41 - .../bin/ruby/shell/commands/remove_peer.rb | 40 - .../ruby/shell/commands/restore_snapshot.rb | 41 - .../test/bin/ruby/shell/commands/revoke.rb | 40 - .../src/test/bin/ruby/shell/commands/scan.rb | 76 -- .../bin/ruby/shell/commands/show_filters.rb | 55 -- .../test/bin/ruby/shell/commands/snapshot.rb | 37 - .../src/test/bin/ruby/shell/commands/split.rb | 43 - .../ruby/shell/commands/start_replication.rb | 43 - .../test/bin/ruby/shell/commands/status.rb | 41 - .../ruby/shell/commands/stop_replication.rb | 43 - .../test/bin/ruby/shell/commands/truncate.rb | 39 - .../test/bin/ruby/shell/commands/unassign.rb | 44 - .../ruby/shell/commands/user_permission.rb | 47 - .../test/bin/ruby/shell/commands/version.rb | 38 - .../test/bin/ruby/shell/commands/whoami.rb | 37 - .../test/bin/ruby/shell/commands/zk_dump.rb | 35 - .../src/test/bin/ruby/shell/formatter.rb | 171 ---- .../src/test/bin/start-hbase.sh | 55 -- .../src/test/bin/stop-hbase.sh | 73 -- ...titan_customizations_to_hbase_script.patch | 28 - .../bin/titan_customizations_to_hirb.rb.patch | 20 - .../src/test/bin/zookeepers.sh | 61 -- titan-hbase-parent/titan-hbase-096/pom.xml | 210 ----- .../diskstorage/hbase/HBaseAdmin0_96.java | 150 ---- .../diskstorage/hbase/HBaseCompat0_96.java | 43 - .../diskstorage/hbase/HConnection0_96.java | 35 - .../titan/diskstorage/hbase/HTable0_96.java | 46 - .../src/test/bin/get-active-master.rb | 44 - .../src/test/bin/graceful_stop.sh | 134 --- .../titan-hbase-096/src/test/bin/hbase | 381 -------- .../src/test/bin/hbase-cleanup.sh | 139 --- .../src/test/bin/hbase-common.sh | 41 - .../src/test/bin/hbase-config.sh | 155 ---- .../src/test/bin/hbase-daemon.sh | 279 ------ .../src/test/bin/hbase-daemons.sh | 55 -- .../titan-hbase-096/src/test/bin/hbase-jruby | 22 - .../titan-hbase-096/src/test/bin/hirb.rb | 190 ---- .../src/test/bin/local-master-backup.sh | 57 -- .../src/test/bin/local-regionservers.sh | 56 -- .../src/test/bin/master-backup.sh | 76 -- .../src/test/bin/region_mover.rb | 487 ----------- .../src/test/bin/region_status.rb | 153 ---- .../src/test/bin/regionservers.sh | 75 -- .../test/bin/replication/copy_tables_desc.rb | 73 -- .../src/test/bin/rolling-restart.sh | 165 ---- .../src/test/bin/ruby/hbase.rb | 80 -- .../src/test/bin/ruby/hbase/admin.rb | 823 ------------------ .../src/test/bin/ruby/hbase/hbase.rb | 59 -- .../test/bin/ruby/hbase/replication_admin.rb | 76 -- .../src/test/bin/ruby/hbase/security.rb | 233 ----- .../src/test/bin/ruby/hbase/table.rb | 539 ------------ .../src/test/bin/ruby/irb/hirb.rb | 59 -- .../src/test/bin/ruby/shell.rb | 348 -------- .../src/test/bin/ruby/shell/commands.rb | 120 --- .../test/bin/ruby/shell/commands/add_peer.rb | 43 - .../src/test/bin/ruby/shell/commands/alter.rb | 92 -- .../bin/ruby/shell/commands/alter_async.rb | 65 -- .../ruby/shell/commands/alter_namespace.rb | 44 - .../bin/ruby/shell/commands/alter_status.rb | 37 - .../test/bin/ruby/shell/commands/assign.rb | 41 - .../bin/ruby/shell/commands/balance_switch.rb | 42 - .../test/bin/ruby/shell/commands/balancer.rb | 40 - .../shell/commands/catalogjanitor_enabled.rb | 40 - .../ruby/shell/commands/catalogjanitor_run.rb | 37 - .../shell/commands/catalogjanitor_switch.rb | 41 - .../bin/ruby/shell/commands/clone_snapshot.rb | 40 - .../bin/ruby/shell/commands/close_region.rb | 59 -- .../test/bin/ruby/shell/commands/compact.rb | 47 - .../src/test/bin/ruby/shell/commands/count.rb | 76 -- .../test/bin/ruby/shell/commands/create.rb | 68 -- .../ruby/shell/commands/create_namespace.rb | 41 - .../test/bin/ruby/shell/commands/delete.rb | 54 -- .../ruby/shell/commands/delete_snapshot.rb | 37 - .../test/bin/ruby/shell/commands/deleteall.rb | 57 -- .../test/bin/ruby/shell/commands/describe.rb | 41 - .../ruby/shell/commands/describe_namespace.rb | 41 - .../test/bin/ruby/shell/commands/disable.rb | 36 - .../bin/ruby/shell/commands/disable_all.rb | 49 -- .../bin/ruby/shell/commands/disable_peer.rb | 41 - .../src/test/bin/ruby/shell/commands/drop.rb | 36 - .../test/bin/ruby/shell/commands/drop_all.rb | 49 -- .../bin/ruby/shell/commands/drop_namespace.rb | 36 - .../test/bin/ruby/shell/commands/enable.rb | 36 - .../bin/ruby/shell/commands/enable_all.rb | 49 -- .../bin/ruby/shell/commands/enable_peer.rb | 41 - .../test/bin/ruby/shell/commands/exists.rb | 38 - .../src/test/bin/ruby/shell/commands/flush.rb | 41 - .../src/test/bin/ruby/shell/commands/get.rb | 91 -- .../bin/ruby/shell/commands/get_counter.rb | 53 -- .../test/bin/ruby/shell/commands/get_table.rb | 46 - .../src/test/bin/ruby/shell/commands/grant.rb | 44 - .../test/bin/ruby/shell/commands/hlog_roll.rb | 39 - .../src/test/bin/ruby/shell/commands/incr.rb | 55 -- .../bin/ruby/shell/commands/is_disabled.rb | 38 - .../bin/ruby/shell/commands/is_enabled.rb | 38 - .../src/test/bin/ruby/shell/commands/list.rb | 47 - .../bin/ruby/shell/commands/list_namespace.rb | 47 - .../shell/commands/list_namespace_tables.rb | 45 - .../bin/ruby/shell/commands/list_peers.rb | 47 - .../shell/commands/list_replicated_tables.rb | 52 -- .../bin/ruby/shell/commands/list_snapshots.rb | 52 -- .../bin/ruby/shell/commands/major_compact.rb | 48 - .../bin/ruby/shell/commands/merge_region.rb | 49 -- .../src/test/bin/ruby/shell/commands/move.rb | 47 - .../src/test/bin/ruby/shell/commands/put.rb | 52 -- .../bin/ruby/shell/commands/remove_peer.rb | 39 - .../ruby/shell/commands/rename_snapshot.rb | 37 - .../ruby/shell/commands/restore_snapshot.rb | 41 - .../test/bin/ruby/shell/commands/revoke.rb | 40 - .../src/test/bin/ruby/shell/commands/scan.rb | 110 --- .../bin/ruby/shell/commands/show_filters.rb | 54 -- .../test/bin/ruby/shell/commands/snapshot.rb | 37 - .../src/test/bin/ruby/shell/commands/split.rb | 42 - .../test/bin/ruby/shell/commands/status.rb | 40 - .../bin/ruby/shell/commands/table_help.rb | 33 - .../src/test/bin/ruby/shell/commands/trace.rb | 76 -- .../test/bin/ruby/shell/commands/truncate.rb | 38 - .../ruby/shell/commands/truncate_preserve.rb | 38 - .../test/bin/ruby/shell/commands/unassign.rb | 45 - .../ruby/shell/commands/user_permission.rb | 47 - .../test/bin/ruby/shell/commands/version.rb | 37 - .../test/bin/ruby/shell/commands/whoami.rb | 37 - .../test/bin/ruby/shell/commands/zk_dump.rb | 34 - .../src/test/bin/ruby/shell/formatter.rb | 170 ---- .../src/test/bin/start-hbase.sh | 63 -- .../src/test/bin/stop-hbase.sh | 67 -- .../test/bin/test/process_based_cluster.sh | 110 --- ...titan_customizations_to_hbase_script.patch | 46 - .../src/test/bin/zookeepers.sh | 61 -- 202 files changed, 1 insertion(+), 16868 deletions(-) delete mode 100644 titan-hbase-parent/titan-hbase-094/.gitignore delete mode 100644 titan-hbase-parent/titan-hbase-094/pom.xml delete mode 100644 titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java delete mode 100644 titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java delete mode 100644 titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java delete mode 100644 titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java delete mode 100644 titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hbase delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/local-master-backup.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch delete mode 100644 titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch delete mode 100755 titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh delete mode 100644 titan-hbase-parent/titan-hbase-096/pom.xml delete mode 100644 titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java delete mode 100644 titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java delete mode 100644 titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java delete mode 100644 titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/local-master-backup.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh delete mode 100644 titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch delete mode 100755 titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh diff --git a/pom.xml b/pom.xml index a7806470a1..4d19560aaf 100644 --- a/pom.xml +++ b/pom.xml @@ -71,10 +71,7 @@ 4.4.1 1.2.1 2.7.1 - 0.94.25 - 0.96.2 - ${hbase096.core.version}-hadoop2 - 0.98.2 + 0.98.16.1 ${hbase098.core.version}-hadoop2 1.0.2 ${hbase100.core.version} diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 0b0010d6da..31d00b0445 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -13,8 +13,6 @@ titan-hbase-core - titan-hbase-094 - titan-hbase-096 titan-hbase-098 titan-hbase-10 titan-hbase diff --git a/titan-hbase-parent/titan-hbase-094/.gitignore b/titan-hbase-parent/titan-hbase-094/.gitignore deleted file mode 100644 index 89540d8235..0000000000 --- a/titan-hbase-parent/titan-hbase-094/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/dependency-reduced-pom.xml diff --git a/titan-hbase-parent/titan-hbase-094/pom.xml b/titan-hbase-parent/titan-hbase-094/pom.xml deleted file mode 100644 index f176e66a86..0000000000 --- a/titan-hbase-parent/titan-hbase-094/pom.xml +++ /dev/null @@ -1,352 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-hbase-parent - 1.0.1-SNAPSHOT - ../pom.xml - - titan-hbase-094 - Titan-HBase: 0.94 Compatibility Shim - http://thinkaurelius.github.com/titan/ - - - ${hadoop1.version} - 1.6.5 - 1.0 - ${project.parent.parent.basedir} - - - - - ${project.groupId} - titan-hbase-core - ${project.version} - - - commons-net - commons-net - - - - - ${project.groupId} - titan-hbase-core - ${project.version} - tests - test - - - org.apache.hbase - hbase - ${hbase094.version} - true - - - avro - org.apache.avro - - - jruby-complete - org.jruby - - - asm - asm - - - - - com.google.protobuf - protobuf-java - true - - - - - - - - - com.google.protobuf - protobuf-java - 2.4.0a - - - - - - src/main/java - - - ${project.basedir}/../common/test/resources - true - - - - - maven-resources-plugin - - - filter-hbase-conf - - - - - maven-dependency-plugin - - - generate-cached-classpath - - - unpack-common-test-classes - - - - - maven-assembly-plugin - - - copy-hbase-bin - - - - - maven-clean-plugin - - - clean-hbase-files - - - clean-shade-pom - clean - - clean - - - true - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - - maven-deploy-plugin - - - default-deploy - none - - - - - maven-shade-plugin - 2.2 - - - - shade-hbase-094-libs - package - - shade - - - true - false - false - - false - - - com.google.protobuf:protobuf-java - - - - - - com.google.protobuf - com.google.protobuf.protobuf240a - - - - - - - - - - - - - - hadoop1 - - - - !hadoop.profile - - - - - - org.apache.hadoop - hadoop-core - ${hadoop1.version} - true - - - jruby-complete - org.jruby - ${jruby.version} - test - - - - - - hadoop2 - - - - hadoop.profile - 2.0 - - - - - ${hadoop2.version} - - - - - org.apache.hadoop - hadoop-annotations - ${hadoop2.version} - - - org.apache.hadoop - hadoop-hdfs - ${hadoop2.version} - - - org.apache.hadoop - hadoop-client - ${hadoop2.version} - - - - - - mapr - - - - hadoop.profile - mapr-1.0 - - - - - 0.94.12-mapr-1310 - 1.0.3-mapr-3.0.2 - 3.4.5 - -Dzookeeper.version=${zookeeper.version} - - - - mapr-releases - http://repository.mapr.com/maven/ - - false - - - true - - - - - - com.mapr.fs - mapr-hbase - ${hadoop.version} - - - - - - org.apache.hbase - hbase - ${hbase094.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - commons-configuration - commons-configuration - 1.8 - - - org.mortbay.jetty - jetty-util - 6.1.26 - - - - - - diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java deleted file mode 100644 index 5684a5abe0..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/google/common/io/NullOutputStream.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2004 The Guava Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.common.io; - -import com.google.common.annotations.Beta; - -import java.io.OutputStream; - -/** - * Implementation of {@link OutputStream} that simply discards written bytes. - * - * @author Spencer Kimball - * @since 1.0 - */ -@Beta -public final class NullOutputStream extends OutputStream { - /** Discards the specified byte. */ - @Override public void write(int b) { - } - - /** Discards the specified byte array. */ - @Override public void write(byte[] b, int off, int len) { - } -} - diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java deleted file mode 100644 index fd3726806e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_94.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; - -public class HBaseAdmin0_94 implements AdminMask -{ - - private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_94.class); - - private final HBaseAdmin adm; - - public HBaseAdmin0_94(HBaseAdmin adm) - { - this.adm = adm; - } - - @Override - public void clearTable(String tableName, long timestamp) throws IOException - { - if (!adm.tableExists(tableName)) { - log.debug("clearStorage() called before table {} was created, skipping.", tableName); - return; - } - -// long before = System.currentTimeMillis(); -// try { -// adm.disableTable(tableName); -// adm.deleteTable(tableName); -// } catch (IOException e) { -// throw new PermanentBackendException(e); -// } -// ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); -// long after = System.currentTimeMillis(); -// logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before); - - - // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than - // disabling and deleting tables. - HTable table = null; - - try { - table = new HTable(adm.getConfiguration(), tableName); - - Scan scan = new Scan(); - scan.setBatch(100); - scan.setCacheBlocks(false); - scan.setCaching(2000); - scan.setTimeRange(0, Long.MAX_VALUE); - scan.setMaxVersions(1); - - ResultScanner scanner = null; - - try { - scanner = table.getScanner(scan); - - for (Result res : scanner) { - Delete d = new Delete(res.getRow()); - - d.setTimestamp(timestamp); - table.delete(d); - } - } finally { - IOUtils.closeQuietly(scanner); - } - } finally { - IOUtils.closeQuietly(table); - } - } - - @Override - public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException - { - return adm.getTableDescriptor(tableName.getBytes()); - } - - @Override - public boolean tableExists(String tableName) throws IOException - { - return adm.tableExists(tableName); - } - - @Override - public void createTable(HTableDescriptor desc) throws IOException - { - adm.createTable(desc); - } - - @Override - public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException - { - adm.createTable(desc, startKey, endKey, numRegions); - } - - @Override - public int getEstimatedRegionServerCount() - { - int serverCount = -1; - try { - serverCount = adm.getClusterStatus().getServers().size(); - log.debug("Read {} servers from HBase ClusterStatus", serverCount); - } catch (IOException e) { - log.debug("Unable to retrieve HBase cluster status", e); - } - return serverCount; - } - - @Override - public void disableTable(String tableName) throws IOException - { - adm.disableTable(tableName); - } - - @Override - public void enableTable(String tableName) throws IOException - { - adm.enableTable(tableName); - } - - @Override - public boolean isTableDisabled(String tableName) throws IOException - { - return adm.isTableDisabled(tableName); - } - - @Override - public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException - { - adm.addColumn(tableName, columnDescriptor); - } - - @Override - public void close() throws IOException - { - adm.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java deleted file mode 100644 index 26238c1d16..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_94.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.io.hfile.Compression; - -public class HBaseCompat0_94 implements HBaseCompat { - - @Override - public void setCompression(HColumnDescriptor cd, String algo) { - cd.setCompressionType(Compression.Algorithm.valueOf(algo)); - } - - @Override - public HTableDescriptor newTableDescriptor(String tableName) { - return new HTableDescriptor(tableName); - } - - @Override - public ConnectionMask createConnection(Configuration conf) throws IOException - { - return new HConnection0_94(HConnectionManager.createConnection(conf)); - } - - @Override - public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc) - { - tdesc.addFamily(cdesc); - } - - @Override - public void setTimestamp(Delete d, long timestamp) - { - d.setTimestamp(timestamp); - } - -} \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java deleted file mode 100644 index ed55390b80..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_94.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; - -public class HConnection0_94 implements ConnectionMask -{ - - private final HConnection cnx; - - public HConnection0_94(HConnection cnx) - { - this.cnx = cnx; - } - - @Override - public TableMask getTable(String name) throws IOException - { - return new HTable0_94(cnx.getTable(name)); - } - - @Override - public AdminMask getAdmin() throws IOException - { - return new HBaseAdmin0_94(new HBaseAdmin(cnx)); - } - - @Override - public void close() throws IOException - { - cnx.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java b/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java deleted file mode 100644 index 719a48e136..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_94.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Row; -import org.apache.hadoop.hbase.client.Scan; - -public class HTable0_94 implements TableMask -{ - private final HTableInterface table; - - public HTable0_94(HTableInterface table) - { - this.table = table; - } - - @Override - public ResultScanner getScanner(Scan filter) throws IOException - { - return table.getScanner(filter); - } - - @Override - public Result[] get(List gets) throws IOException - { - return table.get(gets); - } - - @Override - public void batch(List writes, Object[] results) throws IOException, InterruptedException - { - table.batch(writes, results); - table.flushCommits(); - } - - @Override - public void close() throws IOException - { - table.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb deleted file mode 100755 index 8887a4574c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/get-active-master.rb +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env hbase-jruby -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Prints the hostname of the machine running the active master. - -include Java -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.ServerName -import org.apache.hadoop.hbase.zookeeper.ZKUtil -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher - -# disable debug/info logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) -org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) - -config = HBaseConfiguration.create - -zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) -begin - master_address = ZKUtil.getData(zk, zk.masterAddressZNode) - if master_address - puts ServerName.parseVersionedServerName(master_address).getHostname() - else - puts 'Master not running' - end -ensure - zk.close() -end - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh deleted file mode 100755 index 9a62440064..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/graceful_stop.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2011 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Move regions off a server then stop it. Optionally restart and reload. -# Turn off the balancer before running this script. -function usage { - echo "Usage: graceful_stop.sh [--config ] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " debug Print helpful debug information" - echo " hostname Hostname of server we are to stop" - exit 1 -} - -if [ $# -lt 1 ]; then - usage -fi - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh -# Get arguments -restart= -reload= -debug= -thrift= -rest= -while [ $# -gt 0 ] -do - case "$1" in - --thrift) thrift=true; shift;; - --rest) rest=true; shift;; - --restart) restart=true; shift;; - --reload) reload=true; shift;; - --debug) debug="--debug"; shift;; - --) shift; break;; - -*) usage ;; - *) break;; # terminate while loop - esac -done - -# "$@" contains the rest. Must be at least the hostname left. -if [ $# -lt 1 ]; then - usage -fi - -hostname=$1 -filename="/tmp/$hostname" -# Run the region mover script. -echo "Disabling balancer! (if required)" -HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` -echo "Previous balancer state was $HBASE_BALANCER_STATE" -echo "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname -echo "Unloaded $hostname region(s)" -# Stop the server. Have to put hostname into its own little file for hbase-daemons.sh -hosts="/tmp/$(basename $0).$$.tmp" -echo $hostname >> $hosts -if [ "$thrift" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift -fi -if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest -fi -"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver -if [ "$restart" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver - if [ "$thrift" != "" ]; then - # -b 0.0.0.0 says listen on all interfaces rather than just default. - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 - fi - if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest - fi - if [ "$reload" != "" ]; then - echo "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname - echo "Reloaded $hostname region(s)" - fi -fi - -if [ $HBASE_BALANCER_STATE != "false" ]; then - echo "Restoring balancer state to" $HBASE_BALANCER_STATE - echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null -fi - -# Cleanup tmp files. -trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase deleted file mode 100755 index 13c265aea4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase +++ /dev/null @@ -1,373 +0,0 @@ -#! /usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# The hbase command script. Based on the hadoop command script putting -# in hbase classes, libs and configurations ahead of hadoop's. -# -# TODO: Narrow the amount of duplicated code. -# -# Environment Variables: -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HBASE_CLASSPATH Extra Java CLASSPATH entries. -# -# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding -# native libaries. -# -# HBASE_OPTS Extra Java runtime options. -# -# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -# -# HBASE_ROOT_LOGGER The root appender. Default is INFO,console -# -# MAVEN_HOME Where mvn is installed. -# -# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. -# Defaults to the jar packaged with HBase. -# -# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -# Empty by default. -# -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -# Detect if we are in hbase sources dir -in_dev_env=false -if [ -d "${HBASE_HOME}/target" ]; then - in_dev_env=true -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hbase " - echo "where an option from one of these categories:" - echo "" - echo "DBA TOOLS" - echo " shell run the HBase shell" - echo " hbck run the hbase 'fsck' tool" - echo " hlog write-ahead-log analyzer" - echo " hfile store file analyzer" - echo " zkcli run the ZooKeeper shell" - echo "" - echo "PROCESS MANAGEMENT" - echo " master run an HBase HMaster node" - echo " regionserver run an HBase HRegionServer node" - echo " zookeeper run a Zookeeper server" - echo " rest run an HBase REST server" - echo " thrift run the HBase Thrift server" - echo " thrift2 run the HBase Thrift2 server" - echo " avro run an HBase Avro server" - echo "" - echo "PACKAGE MANAGEMENT" - echo " classpath dump hbase CLASSPATH" - echo " version print the version" - echo "" - echo " or" - echo " CLASSNAME run the class named CLASSNAME" - echo "Most commands print help when invoked w/o parameters." - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -MVN="mvn" -if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn -fi - -# override default settings for this command, if applicable -if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then - . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" -fi - -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX -fi - -# so that filenames w/ spaces are handled correctly in loops below -IFS= - -# CLASSPATH initially contains $HBASE_CONF_DIR -CLASSPATH="${HBASE_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -add_maven_deps_to_classpath() { - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - # ------------------- - # Titan build support - # ------------------- - # Check for Maven command line options file. - # If titan-hbase has already been built, then this file will contain options - # to target the version of Hadoop used during compilation. This is - # necessary when the user compiles titan-hbase with a non-default Hadoop - # version (e.g. 2.2.0). If we omitted this check, then we'd try to start - # HBase with the default Hadoop version (currentl 1.0.4), and it would break. - MVN_OPTS= - optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" - unset IFS - if [ -f "${optf}" ] - then - MVN_OPTS=`cat "${optf}"` - fi - ${MVN} -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -add_maven_main_classes_to_classpath() { - if [ -d "$HBASE_HOME/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME/target/classes - fi -} - -add_maven_test_classes_to_classpath() { - # For developers, add hbase classes to CLASSPATH - f="$HBASE_HOME/target/test-classes" - if [ -d "${f}" ]; then - CLASSPATH=${CLASSPATH}:${f} - fi -} - -# Add maven target directory -if $in_dev_env; then - add_maven_deps_to_classpath - add_maven_main_classes_to_classpath - add_maven_test_classes_to_classpath -fi - -# For releases, add hbase & webapps to CLASSPATH -# Webapps must come first else it messes up Jetty -if [ -d "$HBASE_HOME/hbase-webapps" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME -fi -if [ -d "$HBASE_HOME/target/hbase-webapps" ]; then - CLASSPATH="${CLASSPATH}:${HBASE_HOME}/target" -fi -for f in $HBASE_HOME/hbase*.jar; do - if [[ $f = *sources.jar ]] - then - : # Skip sources.jar - elif [ -f $f ] - then - CLASSPATH=${CLASSPATH}:$f; - fi -done - -# Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# Add user-specified CLASSPATH first -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${HBASE_CLASSPATH}:${CLASSPATH} -fi - -# default log directory & file -if [ "$HBASE_LOG_DIR" = "" ]; then - HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -if [ "$HBASE_LOGFILE" = "" ]; then - HBASE_LOGFILE='hbase.log' -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - -function append_path() { - if [ -z "$1" ]; then - echo $2 - else - echo $1:$2 - fi -} - -JAVA_PLATFORM="" - -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) -if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) -fi - -if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then - if [ -z $JAVA_PLATFORM ]; then - JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - fi - if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) - fi - - if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) - fi -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -# restore ordinary behaviour -unset IFS - -#Set the right GC options based on the what we are running -declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") -for cmd in ${server_cmds[@]}; do - if [[ $cmd == $COMMAND ]]; then - server=true - break - fi -done - -if [[ $server ]]; then - HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" -else - HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" -fi - -# figure out which class to run -if [ "$COMMAND" = "shell" ] ; then - # eg export JRUBY_HOME=/usr/local/share/jruby - if [ "$JRUBY_HOME" != "" ] ; then - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" -elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -elif [ "$COMMAND" = "hlog" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' -elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFile' -elif [ "$COMMAND" = "zkcli" ] ; then - # ZooKeeperMainServerArg returns '-server HOST:PORT' or empty string. - SERVER_ARG=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServerArg` - CLASS="org.apache.zookeeper.ZooKeeperMain ${SERVER_ARG}" - -elif [ "$COMMAND" = "master" ] ; then - CLASS='org.apache.hadoop.hbase.master.HMaster' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" - fi -elif [ "$COMMAND" = "regionserver" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" - fi -elif [ "$COMMAND" = "thrift" ] ; then - CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "thrift2" ] ; then - CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "rest" ] ; then - CLASS='org.apache.hadoop.hbase.rest.Main' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" - fi -elif [ "$COMMAND" = "avro" ] ; then - CLASS='org.apache.hadoop.hbase.avro.AvroServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_AVRO_OPTS" - fi -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" - fi - -elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit 0 -elif [ "$COMMAND" = "version" ] ; then - CLASS='org.apache.hadoop.hbase.util.VersionInfo' -else - CLASS=$COMMAND -fi - -# Have JVM dump heap if we run out of memory. Files will be 'launch directory' -# and are named like the following: java_pid21612.hprof. Apparently it doesn't -# 'cost' to have this flag enabled. Its a 1.6 flag only. See: -# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" -HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" -HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" -HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" -fi - -# Enable security logging on the master and regionserver only -if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}" -else - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" -fi - -# Exec unless HBASE_NOEXEC is set. -if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -else - exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh deleted file mode 100755 index 966744ae52..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-config.sh +++ /dev/null @@ -1,120 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# included in all the hbase scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* -# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -# resolve links - "${BASH_SOURCE-$0}" may be a softlink - -this="${BASH_SOURCE-$0}" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin">/dev/null; pwd` -this="$bin/$script" - -# the root of the hbase installation -if [ -z "$HBASE_HOME" ]; then - export HBASE_HOME=`dirname "$this"`/.. -fi - -#check to see if the conf dir or hbase home are given as an optional arguments -while [ $# -gt 1 ] -do - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HBASE_CONF_DIR=$confdir - elif [ "--hosts" = "$1" ] - then - shift - hosts=$1 - shift - HBASE_REGIONSERVERS=$hosts - else - # Presume we are at end of options and break - break - fi -done - -# Allow alternate hbase conf dir location. -HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" -# List of hbase regions servers. -HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" -# List of hbase secondary masters. -HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" - -# Source the hbase-env.sh. Will have JAVA_HOME defined. -# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. -if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then - . "${HBASE_CONF_DIR}/hbase-env.sh" - export HBASE_ENV_INIT="true" -fi - -# Newer versions of glibc use an arena memory allocator that causes virtual -# memory usage to explode. Tune the variable down to prevent vmem explosion. -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} - -if [ -z "$JAVA_HOME" ]; then - for candidate in \ - /usr/lib/jvm/java-6-sun \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ - /usr/lib/j2sdk1.6-sun \ - /usr/java/jdk1.6* \ - /usr/java/jre1.6* \ - /Library/Java/Home ; do - if [ -e $candidate/bin/java ]; then - export JAVA_HOME=$candidate - break - fi - done - # if we didn't set it - if [ -z "$JAVA_HOME" ]; then - cat 1>&2 < http://java.sun.com/javase/downloads/ < | -| | -| HBase requires Java 1.6 or later. | -| NOTE: This script will find Sun Java whether you install using the | -| binary or the RPM based installer. | -+======================================================================+ -EOF - exit 1 - fi -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh deleted file mode 100755 index 6f680905cd..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemon.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Runs a Hadoop hbase command as a daemon. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_LOG_DIR Where log files are stored. PWD by default. -# HBASE_PID_DIR The pid files are stored. /tmp by default. -# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default -# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. -# -# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh - -usage="Usage: hbase-daemon.sh [--config ]\ - (start|stop|restart) \ - " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# get arguments -startStop=$1 -shift - -command=$1 -shift - -hbase_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" - num=$prev - done - mv -f "$log" "$log.$num"; - fi -} - -wait_until_done () -{ - p=$1 - cnt=${HBASE_SLAVE_TIMEOUT:-300} - origcnt=$cnt - while kill -0 $p > /dev/null 2>&1; do - if [ $cnt -gt 1 ]; then - cnt=`expr $cnt - 1` - sleep 1 - else - echo "Process did not complete after $origcnt seconds, killing." - kill -9 $p - exit 1 - fi - done - return 0 -} - -# get log directory -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_PID_DIR" = "" ]; then - HBASE_PID_DIR=/tmp -fi - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -# Some variables -# Work out java location so can print version into log. -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi -JAVA=$JAVA_HOME/bin/java -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -export HBASE_ROOT_LOGGER="INFO,DRFA" -export HBASE_SECURITY_LOGGER="INFO,DRFAS" -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid - -if [ -n "$SERVER_GC_OPTS" ]; then - export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi -if [ -n "$CLIENT_GC_OPTS" ]; then - export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi - -# Set default scheduling priority -if [ "$HBASE_NICENESS" = "" ]; then - export HBASE_NICENESS=0 -fi - -case $startStop in - - (start) - mkdir -p "$HBASE_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - hbase_rotate_log $logout - hbase_rotate_log $loggc - echo starting $command, logging to $logout - # Add to the command log file vital stats on our environment. - echo "`date` Starting $command on `hostname`" >> $loglog - echo "`ulimit -a`" >> $loglog 2>&1 - nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - $command "$@" $startStop > "$logout" 2>&1 < /dev/null & - echo $! > $pid - sleep 1; head "$logout" - ;; - - (stop) - if [ -f $pid ]; then - # kill -0 == see if the PID exists - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo -n stopping $command - echo "`date` Terminating $command" >> $loglog - kill `cat $pid` > /dev/null 2>&1 - while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; - done - rm $pid - echo - else - retval=$? - echo no $command to stop because kill -0 of pid `cat $pid` failed with status $retval - fi - else - echo no $command to stop because no pid file $pid - fi - ;; - - (restart) - thiscmd=$0 - args=$@ - # stop the command - $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & - wait_until_done $! - # wait a user-specified sleep period - sp=${HBASE_RESTART_SLEEP:-3} - if [ $sp -gt 0 ]; then - sleep $sp - fi - # start the command - $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & - wait_until_done $! - ;; - - (*) - echo $usage - exit 1 - ;; - -esac diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh deleted file mode 100755 index 843eaaa74f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-daemons.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [--hosts regionserversfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. $bin/hbase-config.sh - -remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" -args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" - -command=$2 -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (master-backup) - exec "$bin/master-backup.sh" $args - ;; - (*) - exec "$bin/regionservers.sh" $args - ;; -esac - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby b/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby deleted file mode 100755 index 37bce46f6b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hbase-jruby +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - - -`dirname $0`/hbase org.jruby.Main $* - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb deleted file mode 100755 index c7925150cb..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/hirb.rb +++ /dev/null @@ -1,185 +0,0 @@ -# -# Copyright 2009 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports -# and hbase commands and then loads jirb. Outputs a banner that tells user -# where to find help, shell version, and loads up a custom hirb. - -# TODO: Add 'debug' support (client-side logs show in shell). Add it as -# command-line option and as command. -# TODO: Interrupt a table creation or a connection to a bad master. Currently -# has to time out. Below we've set down the retries for rpc and hbase but -# still can be annoying (And there seem to be times when we'll retry for -# ever regardless) -# TODO: Add support for listing and manipulating catalog tables, etc. -# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes - -# Run the java magic include and import basic HBase types that will help ease -# hbase hacking. -include Java - -# Some goodies for hirb. Should these be left up to the user's discretion? -require 'irb/completion' - -# Add HBase to the ruby load path -if File.exists?(File.join(File.dirname(__FILE__), "ruby", "hbase.rb")) - $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "ruby") -end - -# -# FIXME: Switch args processing to getopt -# -# See if there are args for this shell. If any, read and then strip from ARGV -# so they don't go through to irb. Output shell 'usage' if user types '--help' -cmdline_help = </dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1" - exit -fi - -# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] -export HBASE_MASTER_OPTS=" " - -run_master () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_MASTER_ARGS="\ - -D hbase.master.port=`expr 60000 + $DN` \ - -D hbase.master.info.port=`expr 60010 + $DN` \ - --backup" - "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_master $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh deleted file mode 100755 index a4d5a1d932..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/local-regionservers.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple regionservers on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 100 regionservers (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1 2" - exit -fi - -# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] -export HBASE_REGIONSERVER_OPTS=" " - -run_regionserver () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 60200 + $DN` \ - -D hbase.regionserver.info.port=`expr 60300 + $DN`" - "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_regionserver $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh deleted file mode 100755 index 114757c763..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/master-backup.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2010 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all backup master hosts. -# -# Environment Variables -# -# HBASE_BACKUP_MASTERS File naming remote hosts. -# Default is ${HBASE_CONF_DIR}/backup-masters -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the master backup file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_BACKUP_MASTERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_BACKUP_MASTERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" - else - export HOSTLIST="${HBASE_BACKUP_MASTERS}" - fi -fi - - -args=${@// /\\ } -args=${args/master-backup/master} - -if [ -f $HOSTLIST ]; then - for hmaster in `cat "$HOSTLIST"`; do - ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ - 2>&1 | sed "s/^/$hmaster: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb deleted file mode 100755 index 5c6bea0ae1..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_mover.rb +++ /dev/null @@ -1,475 +0,0 @@ -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Moves regions. Will confirm region access in current location and will -# not move a new region until successful confirm of region loading in new -# location. Presumes balancer is disabled when we run (not harmful if its -# on but this script and balancer will end up fighting each other). -# Does not work for case of multiple regionservers all running on the -# one node. -require 'optparse' -include Java -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.Get -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.HConnectionManager -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; -import org.apache.hadoop.hbase.HServerAddress -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.util.Writables -import org.apache.hadoop.conf.Configuration -import org.apache.commons.logging.Log -import org.apache.commons.logging.LogFactory - -# Name of this script -NAME = "region_mover" - -# Get root table reference -def getRootTable(config) - # Keep meta reference in ruby global - if not $ROOT - $ROOT = HTable.new(config, HConstants::ROOT_TABLE_NAME) - end - return $ROOT -end - -# Get meta table reference -def getMetaTable(config) - # Keep meta reference in ruby global - if not $META - $META = HTable.new(config, HConstants::META_TABLE_NAME) - end - return $META -end - -# Get table instance. -# Maintains cache of table instances. -def getTable(config, name) - # Keep dictionary of tables in ruby global - if not $TABLES - $TABLES = {} - end - key = Bytes.toString(name) - if not $TABLES[key] - $TABLES[key] = HTable.new(config, name) - end - return $TABLES[key] -end - - -# Returns true if passed region is still on 'original' when we look at .META. -def isSameServer(admin, r, original) - server = getServerNameForRegion(admin, r) - return false unless server - return true unless original - return server == original -end - -class RubyAbortable - include org.apache.hadoop.hbase.Abortable - def abort(why, e) - puts "ABORTED! why=" + why + ", e=" + e.to_s - end -end - -# Get servername that is up in .META.; this is hostname + port + startcode comma-delimited. -# Can return nil -def getServerNameForRegion(admin, r) - if r.isRootRegion() - # Hack - tracker = org.apache.hadoop.hbase.zookeeper.RootRegionTracker.new(admin.getConnection().getZooKeeperWatcher(), RubyAbortable.new()) - tracker.start() - while not tracker.isLocationAvailable() - sleep 0.1 - end - # Make a fake servername by appending ',' - rootServer = tracker.getRootRegionLocation().toString() + "," - tracker.stop() - return rootServer - end - table = nil - if r.isMetaRegion() - table = getRootTable(admin.getConfiguration()) - else - table = getMetaTable(admin.getConfiguration()) - end - g = Get.new(r.getRegionName()) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - result = table.get(g) - server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - return nil unless server - return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s -end - -# Trys to scan a row from passed region -# Throws exception if can't -def isSuccessfulScan(admin, r) - scan = Scan.new(r.getStartKey()) - scan.setBatch(1) - scan.setCaching(1) - scan.setFilter(FirstKeyOnlyFilter.new()) - table = getTable(admin.getConfiguration(), r.getTableName()) - scanner = table.getScanner(scan) - begin - results = scanner.next() - # We might scan into next region, this might be an empty table. - # But if no exception, presume scanning is working. - ensure - scanner.close() - table.close() - end -end - -# Check region has moved successful and is indeed hosted on another server -# Wait until that is the case. -def move(admin, r, newServer, original) - # Now move it. Do it in a loop so can retry if fail. Have seen issue where - # we tried move region but failed and retry put it back on old location; - # retry in this case. - retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) - count = 0 - same = true - while count < retries and same - if count > 0 - $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s) - end - count = count + 1 - begin - admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) - rescue java.lang.reflect.UndeclaredThrowableException => e - $LOG.info("Exception moving " + r.getEncodedName() + - "; split/moved? Continuing: " + e) - return - end - # Wait till its up on new server before moving on - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.move.wait.max", 60) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - same = isSameServer(admin, r, original) - break unless same - sleep 0.1 - end - end - raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same - # Assert can Scan from new location. - isSuccessfulScan(admin, r) -end - -# Return the hostname portion of a servername (all up to first ',') -def getHostnamePortFromServerName(serverName) - parts = serverName.split(',') - return parts[0] + ":" + parts[1] -end - -# Return the hostname:port out of a servername (all up to first ',') -def getHostnameFromServerName(serverName) - return serverName.split(',')[0] -end - -# Return array of servernames where servername is hostname+port+startcode -# comma-delimited -def getServers(admin) - serverInfos = admin.getClusterStatus().getServerInfo() - servers = [] - for server in serverInfos - servers << server.getServerName() - end - return servers -end - -# Remove the servername whose hostname portion matches from the passed -# array of servers. Returns as side-effect the servername removed. -def stripServer(servers, hostname) - count = servers.length - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = servers.delete(server) - end - end - # Check server to exclude is actually present - raise RuntimeError, "Server %s not online" % hostname unless servers.length < count - return servername -end - -# Returns a new serverlist that excludes the servername whose hostname portion -# matches from the passed array of servers. -def stripExcludes(servers, excludefile) - excludes = readExcludes(excludefile) - servers = servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) } - # return updated servers list - return servers -end - - -# Return servername that matches passed hostname -def getServerName(servers, hostname) - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = server - break - end - end - raise ArgumentError, "Server %s not online" % hostname unless servername - return servername -end - -# Create a logger and disable the DEBUG-level annoying client logging -def configureLogging(options) - apacheLogger = LogFactory.getLog(NAME) - # Configure log4j to not spew so much - unless (options[:debug]) - logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase.client") - logger.setLevel(org.apache.log4j.Level::INFO) - end - return apacheLogger -end - -# Get configuration instance -def getConfiguration() - config = HBaseConfiguration.create() - # No prefetching on .META. - config.setInt("hbase.client.prefetch.limit", 1) - # Make a config that retries at short intervals many times - config.setInt("hbase.client.pause", 500) - config.setInt("hbase.client.retries.number", 100) - return config -end - -# Now get list of regions on targetServer -def getRegions(config, servername) - connection = HConnectionManager::getConnection(config) - parts = servername.split(',') - rs = connection.getHRegionConnection(parts[0], parts[1].to_i) - return rs.getOnlineRegions() -end - -def deleteFile(filename) - f = java.io.File.new(filename) - f.delete() if f.exists() -end - -# Write HRegionInfo to file -# Need to serialize in case non-printable characters. -# Format is count of regionnames followed by serialized regionnames. -def writeFile(filename, regions) - fos = java.io.FileOutputStream.new(filename) - dos = java.io.DataOutputStream.new(fos) - # Write out a count of region names - dos.writeInt(regions.size()) - # Write actual region names. - for r in regions - bytes = Writables.getBytes(r) - Bytes.writeByteArray(dos, bytes) - end - dos.close() -end - -# See writeFile above. -# Returns array of HRegionInfos -def readFile(filename) - f = java.io.File.new(filename) - return java.util.ArrayList.new() unless f.exists() - fis = java.io.FileInputStream.new(f) - dis = java.io.DataInputStream.new(fis) - # Read count of regions - count = dis.readInt() - regions = java.util.ArrayList.new(count) - index = 0 - while index < count - regions.add(Writables.getHRegionInfo(Bytes.readByteArray(dis))) - index = index + 1 - end - dis.close() - return regions -end - -# Move regions off the passed hostname -def unloadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Clean up any old files. - filename = getFilename(options, hostname) - deleteFile(filename) - # Get an admin instance - admin = HBaseAdmin.new(config) - servers = getServers(admin) - # Remove the server we are unloading from from list of servers. - # Side-effect is the servername that matches this hostname - servername = stripServer(servers, hostname) - - # Remove the servers in our exclude list from list of servers. - servers = stripExcludes(servers, options[:excludesFile]) - puts "Valid region move targets: ", servers - movedRegions = java.util.ArrayList.new() - while true - rs = getRegions(config, servername) - break if rs.length == 0 - count = 0 - $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + - " during this cycle"); - for r in rs - # Get a random server to move the region to. - server = servers[rand(servers.length)] - $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + - " of " + rs.length.to_s + ") to server=" + server); - count = count + 1 - # Assert we can scan region in its current location - isSuccessfulScan(admin, r) - # Now move it. - move(admin, r, server, servername) - movedRegions.add(r) - end - end - if movedRegions.size() > 0 - # Write out file of regions moved - writeFile(filename, movedRegions) - $LOG.info("Wrote list of moved regions to " + filename) - end -end - -# Move regions to the passed hostname -def loadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Get an admin instance - admin = HBaseAdmin.new(config) - filename = getFilename(options, hostname) - regions = readFile(filename) - return if regions.isEmpty() - servername = nil - # Wait till server is up - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.serverstart.wait.max", 180) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - servers = getServers(admin) - begin - servername = getServerName(servers, hostname) - rescue ArgumentError => e - $LOG.info("hostname=" + hostname.to_s + " is not up yet, waiting"); - end - break if servername - sleep 0.5 - end - $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) - count = 0 - for r in regions - exists = false - begin - isSuccessfulScan(admin, r) - exists = true - rescue org.apache.hadoop.hbase.NotServingRegionException => e - $LOG.info("Failed scan of " + e.message) - end - count = count + 1 - next unless exists - currentServer = getServerNameForRegion(admin, r) - if currentServer and currentServer == servername - $LOG.info("Region " + r.getRegionNameAsString() + " (" + count.to_s + - " of " + regions.length.to_s + ") already on target server=" + servername) - next - end - $LOG.info("Moving region " + r.getEncodedName() + " (" + count.to_s + - " of " + regions.length.to_s + ") to server=" + servername); - move(admin, r, servername, currentServer) - end -end - -# Returns an array of hosts to exclude as region move targets -def readExcludes(filename) - if filename == nil - return java.util.ArrayList.new() - end - if ! File.exist?(filename) - puts "Error: Unable to read host exclude file: ", filename - raise RuntimeError - end - - f = File.new(filename, "r") - # Read excluded hosts list - excludes = java.util.ArrayList.new() - while (line = f.gets) - line.strip! # do an inplace drop of pre and post whitespaces - excludes.add(line) unless line.empty? # exclude empty lines - end - puts "Excluding hosts as region move targets: ", excludes - f.close - - return excludes -end - -def getFilename(options, targetServer) - filename = options[:file] - if not filename - filename = "/tmp/" + targetServer - end - return filename -end - - -# Do command-line parsing -options = {} -optparse = OptionParser.new do |opts| - opts.banner = "Usage: #{NAME}.rb [options] load|unload " - opts.separator 'Load or unload regions by moving one at a time' - options[:file] = nil - opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| - options[:file] = file - end - opts.on('-h', '--help', 'Display usage information') do - puts opts - exit - end - options[:debug] = false - opts.on('-d', '--debug', 'Display extra debug logging') do - options[:debug] = true - end - opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| - options[:excludesFile] = file - end -end -optparse.parse! - -# Check ARGVs -if ARGV.length < 2 - puts optparse - exit 1 -end -hostname = ARGV[1] -if not hostname - opts optparse - exit 2 -end -# Create a logger and save it to ruby global -$LOG = configureLogging(options) -case ARGV[0] - when 'load' - loadRegions(options, hostname) - when 'unload' - unloadRegions(options, hostname) - else - puts optparse - exit 3 -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb deleted file mode 100755 index 5ccc8c2e6d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/region_status.rb +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# View the current status of all regions on an HBase cluster. This is -# predominantly used to determined if all the regions in META have been -# onlined yet on startup. -# -# To use this script, run: -# -# ${HBASE_HOME}/bin/hbase org.jruby.Main region_status.rb [wait] [--table ] - - -require 'optparse' - -usage = 'Usage : ./hbase org.jruby.Main region_status.rb [wait]' + - '[--table ]\n' -OptionParser.new do |o| - o.banner = usage - o.on('-t', '--table TABLENAME', 'Only process TABLENAME') do |tablename| - $tablename = tablename - end - o.on('-h', '--help', 'Display help message') { puts o; exit } - o.parse! -end - -SHOULD_WAIT = ARGV[0] == 'wait' -if ARGV[0] and not SHOULD_WAIT - print usage - exit 1 -end - - -require 'java' - -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.MasterNotRunningException -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.util.Writables - -# disable debug logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) -org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) - -config = HBaseConfiguration.create -config.set 'fs.default.name', config.get(HConstants::HBASE_DIR) - -# wait until the master is running -admin = nil -while true - begin - admin = HBaseAdmin.new config - break - rescue MasterNotRunningException => e - print 'Waiting for master to start...\n' - sleep 1 - end -end - -meta_count = 0 -server_count = 0 - -# scan META to see how many regions we should have -if $tablename.nil? - scan = Scan.new -else - tableNameMetaPrefix = $tablename + HConstants::META_ROW_DELIMITER.chr - scan = Scan.new( - (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes - ) -end -scan.cache_blocks = false -scan.caching = 10 -scan.setFilter(FirstKeyOnlyFilter.new) -INFO = 'info'.to_java_bytes -REGION_INFO = 'regioninfo'.to_java_bytes -scan.addColumn INFO, REGION_INFO -table = nil -iter = nil -while true - begin - table = HTable.new config, '.META.'.to_java_bytes - scanner = table.getScanner(scan) - iter = scanner.iterator - break - rescue IOException => ioe - print "Exception trying to scan META: #{ioe}" - sleep 1 - end -end -while iter.hasNext - result = iter.next - rowid = Bytes.toString(result.getRow()) - rowidStr = java.lang.String.new(rowid) - if not $tablename.nil? and not rowidStr.startsWith(tableNameMetaPrefix) - # Gone too far, break - break - end - region = Writables.getHRegionInfo result.getValue(INFO, REGION_INFO) - if not region.isOffline - # only include regions that should be online - meta_count += 1 - end -end -scanner.close -# If we're trying to see the status of all HBase tables, we need to include the -# -ROOT- & .META. tables, that are not included in our scan -if $tablename.nil? - meta_count += 2 -end - -# query the master to see how many regions are on region servers -if not $tablename.nil? - $tableq = HTable.new config, $tablename.to_java_bytes -end -while true - if $tablename.nil? - server_count = admin.getClusterStatus().getRegionsCount() - else - server_count = $tableq.getRegionsInfo().size() - end - print "Region Status: #{server_count} / #{meta_count}\n" - if SHOULD_WAIT and server_count < meta_count - #continue this loop until server & meta count match - sleep 10 - else - break - end -end - -exit server_count == meta_count ? 0 : 1 diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh deleted file mode 100755 index 9759f2b00c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/regionservers.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: regionservers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the regionservers file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_REGIONSERVERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_REGIONSERVERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/regionservers" - else - export HOSTLIST="${HBASE_REGIONSERVERS}" - fi -fi - -for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi -done - -wait diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh deleted file mode 100755 index 23f8d325e8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/rolling-restart.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -function usage() { - echo $usage - exit 1 -} - - -RR_RS=1 -RR_MASTER=1 -RR_GRACEFUL=0 - -for x in "$@" ; do - case "$x" in - --rs-only|-r) - RR_RS=1 - RR_MASTER=0 - RR_GRACEFUL=0 - ;; - --master-only) - RR_RS=0 - RR_MASTER=1 - RR_GRACEFUL=0 - ;; - --graceful) - RR_RS=0 - RR_MASTER=0 - RR_GRACEFUL=1 - ;; - *) - echo Bad argument: $x - usage - exit 1 - ;; - esac -done - -# quick function to get a value from the HBase config file -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'false' ]; then - if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then - echo Cant do selective rolling restart if not running distributed - exit 1 - fi - "$bin"/hbase-daemon.sh restart master -else - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - - if [ $RR_MASTER -eq 1 ]; then - # stop all masters before re-start to avoid races for master znode - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - echo -n "Waiting for Master ZNode ${zmaster} to expire" - while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do - echo -n "." - sleep 1 - done - echo #force a newline - - # all masters are down, now restart - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup - - echo "Wait a minute for master to come up join cluster" - sleep 60 - - # Master joing cluster will start in cleaning out regions in transition. - # Wait until the master has cleaned out regions in transition before - # giving it a bunch of work to do; master is vulnerable during startup - zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` - if [ "$zunassigned" == "null" ]; then zunassigned="unassigned"; fi - zunassigned="$zparent/$zunassigned" - echo -n "Waiting for ${zunassigned} to empty" - while true ; do - unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` - if test 0 -eq ${unassigned} - then - break - else - echo -n " ${unassigned}" - fi - sleep 1 - done - fi - - if [ $RR_RS -eq 1 ]; then - # unlike the masters, roll all regionservers one-at-a-time - export HBASE_SLAVE_PARALLEL=false - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" restart regionserver - fi - - if [ $RR_GRACEFUL -eq 1 ]; then - # gracefully restart all online regionservers - zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zkrs" == "null" ]; then zkrs="rs"; fi - zkrs="$zparent/$zkrs" - online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - for rs in $online_regionservers - do - rs_parts=(${rs//,/ }) - hostname=${rs_parts[0]} - echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" - sleep 1 - done - fi -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb deleted file mode 100644 index 4d97cd0948..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase.rb +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# HBase ruby classes. -# Has wrapper classes for org.apache.hadoop.hbase.client.HBaseAdmin -# and for org.apache.hadoop.hbase.client.HTable. Classes take -# Formatters on construction and outputs any results using -# Formatter methods. These classes are only really for use by -# the hirb.rb HBase Shell script; they don't make much sense elsewhere. -# For example, the exists method on Admin class prints to the formatter -# whether the table exists and returns nil regardless. -include Java - -include_class('java.lang.Integer') {|package,name| "J#{name}" } -include_class('java.lang.Long') {|package,name| "J#{name}" } -include_class('java.lang.Boolean') {|package,name| "J#{name}" } - -module HBaseConstants - COLUMN = "COLUMN" - COLUMNS = "COLUMNS" - TIMESTAMP = "TIMESTAMP" - TIMERANGE = "TIMERANGE" - NAME = org.apache.hadoop.hbase.HConstants::NAME - VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS - IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY - CONFIG = org.apache.hadoop.hbase.HConstants::CONFIG - STOPROW = "STOPROW" - STARTROW = "STARTROW" - ENDROW = STOPROW - RAW = "RAW" - LIMIT = "LIMIT" - METHOD = "METHOD" - MAXLENGTH = "MAXLENGTH" - CACHE_BLOCKS = "CACHE_BLOCKS" - REPLICATION_SCOPE = "REPLICATION_SCOPE" - INTERVAL = 'INTERVAL' - CACHE = 'CACHE' - FILTER = 'FILTER' - SPLITS = 'SPLITS' - SPLITS_FILE = 'SPLITS_FILE' - SPLITALGO = 'SPLITALGO' - NUMREGIONS = 'NUMREGIONS' - - # Load constants from hbase java API - def self.promote_constants(constants) - # The constants to import are all in uppercase - constants.each do |c| - next if c =~ /DEFAULT_.*/ || c != c.upcase - next if eval("defined?(#{c})") - eval("#{c} = '#{c}'") - end - end - - promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) - promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) -end - -# Include classes definition -require 'hbase/hbase' -require 'hbase/admin' -require 'hbase/table' -require 'hbase/replication_admin' -require 'hbase/security' diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb deleted file mode 100644 index 4ad3d88c74..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/admin.rb +++ /dev/null @@ -1,667 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java -java_import org.apache.hadoop.hbase.util.Pair -java_import org.apache.hadoop.hbase.util.RegionSplitter - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class Admin - include HBaseConstants - - def initialize(configuration, formatter) - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - connection = @admin.getConnection() - @conf = configuration - @zk_wrapper = connection.getZooKeeperWatcher() - zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() - @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of tables in hbase - def list(regex = ".*") - begin - # Use the old listTables API first for compatibility with older servers - @admin.listTables(regex).map { |t| t.getNameAsString } - rescue => e - # listTables failed, try the new unprivileged getTableNames API if the cause was - # an AccessDeniedException - if e.cause.kind_of? org.apache.hadoop.ipc.RemoteException and e.cause.unwrapRemoteException().kind_of? org.apache.hadoop.hbase.security.AccessDeniedException - @admin.getTableNames(regex) - else - # Not an access control failure, re-raise - raise e - end - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region flush - def flush(table_or_region_name) - @admin.flush(table_or_region_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family compaction - def compact(table_or_region_name, family = nil) - if family == nil - @admin.compact(table_or_region_name) - else - # We are compacting a column family within a region. - @admin.compact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family major compaction - def major_compact(table_or_region_name, family = nil) - if family == nil - @admin.majorCompact(table_or_region_name) - else - # We are major compacting a column family within a region or table. - @admin.majorCompact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a regionserver's HLog roll - def hlog_roll(server_name) - @admin.rollHLogWriter(server_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region split - def split(table_or_region_name, split_point) - if split_point == nil - @admin.split(table_or_region_name) - else - @admin.split(table_or_region_name, split_point) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a cluster balance - # Returns true if balancer ran - def balancer() - @admin.balancer() - end - - #---------------------------------------------------------------------------------------------- - # Enable/disable balancer - # Returns previous balancer switch setting. - def balance_switch(enableDisable) - @admin.setBalancerRunning( - java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false)) - end - - #---------------------------------------------------------------------------------------------- - # Enables a table - def enable(table_name) - tableExists(table_name) - return if enabled?(table_name) - @admin.enableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Enables all tables matching the given regex - def enable_all(regex) - regex = regex.to_s - @admin.enableTables(regex) - end - - #---------------------------------------------------------------------------------------------- - # Disables a table - def disable(table_name) - tableExists(table_name) - return if disabled?(table_name) - @admin.disableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Disables all tables matching the given regex - def disable_all(regex) - regex = regex.to_s - @admin.disableTables(regex).map { |t| t.getNameAsString } - end - - #--------------------------------------------------------------------------------------------- - # Throw exception if table doesn't exist - def tableExists(table_name) - raise ArgumentError, "Table #{table_name} does not exist.'" unless exists?(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table disabled? - def disabled?(table_name) - @admin.isTableDisabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop(table_name) - tableExists(table_name) - raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name) - - @admin.deleteTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop_all(regex) - regex = regex.to_s - failed = @admin.deleteTables(regex).map { |t| t.getNameAsString } - return failed - end - - #---------------------------------------------------------------------------------------------- - # Returns ZooKeeper status dump - def zk_dump - org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) - end - - #---------------------------------------------------------------------------------------------- - # Creates a table - def create(table_name, *args) - # Fail if table name is not a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Flatten params array - args = args.flatten.compact - - # Fail if no column families defined - raise(ArgumentError, "Table must have at least one column family") if args.empty? - - # Start defining the table - htd = org.apache.hadoop.hbase.HTableDescriptor.new(table_name) - splits = nil - # Args are either columns or splits, add them to the table definition - # TODO: add table options support - args.each do |arg| - unless arg.kind_of?(String) || arg.kind_of?(Hash) - raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") - end - - if arg.kind_of?(String) - # the arg is a string, default action is to add a column to the table - htd.addFamily(hcd(arg, htd)) - else - # arg is a hash. 4 possibilities: - if (arg.has_key?(SPLITS) or arg.has_key?(SPLITS_FILE)) - if arg.has_key?(SPLITS_FILE) - unless File.exist?(arg[SPLITS_FILE]) - raise(ArgumentError, "Splits file #{arg[SPLITS_FILE]} doesn't exist") - end - arg[SPLITS] = [] - File.foreach(arg[SPLITS_FILE]) do |line| - arg[SPLITS].push(line.strip()) - end - end - - splits = Java::byte[][arg[SPLITS].size].new - idx = 0 - arg[SPLITS].each do |split| - splits[idx] = split.to_java_bytes - idx = idx + 1 - end - elsif (arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO)) - # (1) deprecated region pre-split API - raise(ArgumentError, "Column family configuration should be specified in a separate clause") if arg.has_key?(NAME) - raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS) - raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO) - raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 - num_regions = arg[NUMREGIONS] - split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) - splits = split_algo.split(JInteger.valueOf(num_regions)) - elsif (method = arg.delete(METHOD)) - # (2) table_att modification - raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' - raise(ArgumentError, "NUMREGIONS & SPLITALGO must both be specified") unless arg.has_key?(NUMREGIONS) == arg.has_key?(split_algo) - htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] - htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] - htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] - if arg[NUMREGIONS] - raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 - num_regions = arg[NUMREGIONS] - split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg[SPLITALGO]) - splits = split_algo.split(JInteger.valueOf(num_regions)) - end - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - htd.setValue(k, v) - end - end - else - # (3) column family spec - descriptor = hcd(arg, htd) - htd.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) if arg[COMPRESSION_COMPACT] - htd.addFamily(hcd(arg, htd)) - end - end - end - - if splits.nil? - # Perform the create table call - @admin.createTable(htd) - else - # Perform the create table call - @admin.createTable(htd, splits) - end - end - - #---------------------------------------------------------------------------------------------- - # Closes a region. - # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). - # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) - def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) - @admin.closeRegion(region_name, server) - end - end - - #---------------------------------------------------------------------------------------------- - #---------------------------------------------------------------------------------------------- - # Assign a region - def assign(region_name) - @admin.assign(region_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Unassign a region - def unassign(region_name, force) - @admin.unassign(region_name.to_java_bytes, java.lang.Boolean::valueOf(force)) - end - - #---------------------------------------------------------------------------------------------- - # Move a region - def move(encoded_region_name, server = nil) - @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) - end - - #---------------------------------------------------------------------------------------------- - # Returns table's structure description - def describe(table_name) - @admin.getTableDescriptor(table_name.to_java_bytes).to_s - end - - #---------------------------------------------------------------------------------------------- - # Truncates table (deletes all records by recreating the table) - def truncate(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - table_description = h_table.getTableDescriptor() - raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) - yield 'Disabling table...' if block_given? - @admin.disableTable(table_name) - - yield 'Dropping table...' if block_given? - @admin.deleteTable(table_name) - - yield 'Creating table...' if block_given? - @admin.createTable(table_description) - end - - # Check the status of alter command (number of regions reopened) - def alter_status(table_name) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - status = Pair.new() - begin - status = @admin.getAlterStatus(table_name.to_java_bytes) - if status.getSecond() != 0 - puts "#{status.getSecond() - status.getFirst()}/#{status.getSecond()} regions updated." - else - puts "All regions updated." - end - sleep 1 - end while status != nil && status.getFirst() != 0 - puts "Done." - end - - #---------------------------------------------------------------------------------------------- - # Change table structure or table options - def alter(table_name, wait = true, *args) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - # There should be at least one argument - raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? - - # Get table descriptor - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - # Process all args - args.each do |arg| - # Normalize args to support column name only alter specs - arg = { NAME => arg } if arg.kind_of?(String) - - # Normalize args to support shortcut delete syntax - arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - - # No method parameter, try to use the args as a column definition - unless method = arg.delete(METHOD) - # Note that we handle owner here, and also below (see (2)) as part of the "METHOD => 'table_att'" table attributes. - # In other words, if OWNER is specified, then METHOD is set to table_att. - # alter 'tablename', {OWNER => 'username'} (that is, METHOD => 'table_att' is not specified). - if arg[OWNER] - htd.setOwnerString(arg[OWNER]) - @admin.modifyTable(table_name.to_java_bytes, htd) - return - end - - descriptor = hcd(arg, htd) - - if arg[COMPRESSION_COMPACT] - descriptor.setValue(COMPRESSION_COMPACT, arg[COMPRESSION_COMPACT]) - end - column_name = descriptor.getNameAsString - - # If column already exist, then try to alter it. Create otherwise. - if htd.hasFamily(column_name.to_java_bytes) - @admin.modifyColumn(table_name, descriptor) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - else - @admin.addColumn(table_name, descriptor) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - end - next - end - - # Delete column family - if method == "delete" - raise(ArgumentError, "NAME parameter missing for delete method") unless arg[NAME] - @admin.deleteColumn(table_name, arg[NAME]) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - next - end - - # Change table attributes - if method == "table_att" - htd.setMaxFileSize(JLong.valueOf(arg[MAX_FILESIZE])) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg[READONLY])) if arg[READONLY] - htd.setMemStoreFlushSize(JLong.valueOf(arg[MEMSTORE_FLUSHSIZE])) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg[DEFERRED_LOG_FLUSH])) if arg[DEFERRED_LOG_FLUSH] - # (2) Here, we handle the alternate syntax of ownership setting, where method => 'table_att' is specified. - htd.setOwnerString(arg[OWNER]) if arg[OWNER] - - # set a coprocessor attribute - if arg.kind_of?(Hash) - arg.each do |key, value| - k = String.new(key) # prepare to strip - k.strip! - - if (k =~ /coprocessor/i) - # validate coprocessor specs - v = String.new(value) - v.strip! - if !(v =~ /^([^\|]*)\|([^\|]+)\|[\s]*([\d]*)[\s]*(\|.*)?$/) - raise ArgumentError, "Coprocessor value doesn't match spec: #{v}" - end - - # generate a coprocessor ordinal by checking max id of existing cps - maxId = 0 - htd.getValues().each do |k1, v1| - attrName = org.apache.hadoop.hbase.util.Bytes.toString(k1.get()) - # a cp key is coprocessor$(\d) - if (attrName =~ /coprocessor\$(\d+)/i) - ids = attrName.scan(/coprocessor\$(\d+)/i) - maxId = ids[0][0].to_i if ids[0][0].to_i > maxId - end - end - maxId += 1 - htd.setValue(k + "\$" + maxId.to_s, value) - end - end - end - - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - htd.setValue(k, v) - end - end - @admin.modifyTable(table_name.to_java_bytes, htd) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - next - end - - # Unset table attributes - if method == "table_att_unset" - if arg.kind_of?(Hash) - if (!arg[NAME]) - next - end - if (htd.getValue(arg[NAME]) == nil) - raise ArgumentError, "Can not find attribute: #{arg[NAME]}" - end - htd.remove(arg[NAME].to_java_bytes) - @admin.modifyTable(table_name.to_java_bytes, htd) - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - end - next - end - - # Unknown method - raise ArgumentError, "Unknown method: #{method}" - end - end - - def status(format) - status = @admin.getClusterStatus() - if format == "detailed" - puts("version %s" % [ status.getHBaseVersion() ]) - # Put regions in transition first because usually empty - puts("%d regionsInTransition" % status.getRegionsInTransition().size()) - for k, v in status.getRegionsInTransition() - puts(" %s" % [v]) - end - master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors()) - if master_coprocs != nil - puts("master coprocessors: %s" % master_coprocs) - end - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - for name, region in status.getLoad(server).getRegionsLoad() - puts(" %s" % [ region.getNameAsString() ]) - puts(" %s" % [ region.toString() ]) - end - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - elsif format == "simple" - load = 0 - regions = 0 - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - load += status.getLoad(server).getNumberOfRequests() - regions += status.getLoad(server).getNumberOfRegions() - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - puts("Aggregate load: %d, regions: %d" % [ load , regions ] ) - else - puts "#{status.getServersSize} servers, #{status.getDeadServers} dead, #{'%.4f' % status.getAverageLoad} average load" - end - end - - #---------------------------------------------------------------------------------------------- - # - # Helper methods - # - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table enabled - def enabled?(table_name) - @admin.isTableEnabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - #Is supplied region name is encoded region name - def closeEncodedRegion?(region_name, server) - @admin.closeRegionWithEncodedRegionName(region_name, server) - end - - #---------------------------------------------------------------------------------------------- - # Return a new HColumnDescriptor made of passed args - def hcd(arg, htd) - # String arg, single parameter constructor - return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.kind_of?(String) - - raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg[NAME] - - family = htd.getFamily(name.to_java_bytes) - # create it if it's a new family - family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) - - family.setBlockCacheEnabled(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) - family.setScope(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) - family.setInMemory(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) - family.setTimeToLive(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::TTL])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) - family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) - family.setEncodeOnDisk(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODE_ON_DISK) - family.setBlocksize(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) - family.setMaxVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) - family.setMinVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) - bloomtype = arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER].upcase - unless org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else - family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.valueOf(bloomtype)) - end - end - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) - compression = arg[org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION].upcase - unless org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.include?(compression) - raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.constants.join(" ")) - else - family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(compression)) - end - end - - if arg[CONFIG] - raise(ArgumentError, "#{CONFIG} must be a Hash type") unless arg.kind_of?(Hash) - for k,v in arg[CONFIG] - v = v.to_s unless v.nil? - family.setValue(k, v) - end - end - return family - end - - #---------------------------------------------------------------------------------------------- - # Enables/disables a region by name - def online(region_name, on_off) - # Open meta table - meta = org.apache.hadoop.hbase.client.HTable.new(org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) - - # Read region info - # FIXME: fail gracefully if can't find the region - region_bytes = region_name.to_java_bytes - g = org.apache.hadoop.hbase.client.Get.new(region_bytes) - g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) - hri_bytes = meta.get(g).value - - # Change region status - hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) - hri.setOffline(on_off) - - # Write it back - put = org.apache.hadoop.hbase.client.Put.new(region_bytes) - put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri)) - meta.put(put) - end - - #---------------------------------------------------------------------------------------------- - # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Restore specified snapshot - def restore_snapshot(snapshot_name) - @admin.restoreSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Create a new table by cloning the snapshot content - def clone_snapshot(snapshot_name, table) - @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Delete specified snapshot - def delete_snapshot(snapshot_name) - @admin.deleteSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of snapshots - def list_snapshot - @admin.listSnapshots - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb deleted file mode 100644 index 2c37840dd2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/hbase.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -require 'hbase/admin' -require 'hbase/table' -require 'hbase/security' - -module Hbase - class Hbase - attr_accessor :configuration - - def initialize(config = nil) - # Create configuration - if config - self.configuration = config - else - self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create - # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. - configuration.setInt("hbase.client.retries.number", 7) - configuration.setInt("ipc.client.connect.max.retries", 3) - end - end - - def admin(formatter) - ::Hbase::Admin.new(configuration, formatter) - end - - # Create new one each time - def table(table, formatter) - ::Hbase::Table.new(configuration, table, formatter) - end - - def replication_admin(formatter) - ::Hbase::RepAdmin.new(configuration, formatter) - end - - def security_admin(formatter) - ::Hbase::SecurityAdmin.new(configuration, formatter) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb deleted file mode 100644 index 99f0479b0e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/replication_admin.rb +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class RepAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @replication_admin = org.apache.hadoop.hbase.client.replication.ReplicationAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Add a new peer cluster to replicate to - def add_peer(id, cluster_key) - @replication_admin.addPeer(id, cluster_key) - end - - #---------------------------------------------------------------------------------------------- - # Remove a peer cluster, stops the replication - def remove_peer(id) - @replication_admin.removePeer(id) - end - - #--------------------------------------------------------------------------------------------- - # Show replcated tables/column families, and their ReplicationType - def list_replicated_tables - @replication_admin.listReplicated() - end - - #---------------------------------------------------------------------------------------------- - # List all peer clusters - def list_peers - @replication_admin.listPeers - end - - #---------------------------------------------------------------------------------------------- - # Get peer cluster state - def get_peer_state(id) - @replication_admin.getPeerState(id) - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication stream to the specified peer - def enable_peer(id) - @replication_admin.enablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Stop the replication stream to the specified peer - def disable_peer(id) - @replication_admin.disablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication, in an unknown state - def start_replication - @replication_admin.setReplicating(true) - end - - #---------------------------------------------------------------------------------------------- - # Kill switch for replication, stops all its features - def stop_replication - @replication_admin.setReplicating(false) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb deleted file mode 100644 index 5041ec4dbb..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/security.rb +++ /dev/null @@ -1,188 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class SecurityAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @config = configuration - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - # Verify that the specified permission is valid - if (permissions == nil || permissions.length == 0) - raise(ArgumentError, "Invalid permission: no actions associated with user") - end - - if (table_name != nil) - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - if (family != nil) - raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - # invoke cp endpoint to perform access controlse - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, table_name.to_java_bytes, - fambytes, qualbytes, permissions.to_java_bytes) - else - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, permissions.to_java_bytes) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - begin - protocol.grant(user_permission) - rescue java.io.IOException => e - if !(e.message.include? "java.lang.NoSuchMethodException") - raise e - end - - # Server has not the new API, try the old one - if (table_name == nil) - raise "Global permissions not supported by HBase Server" - end - - tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, permissions.to_java_bytes) - protocol.grant(user.to_java_bytes, tp) - end - end - - #---------------------------------------------------------------------------------------------- - def revoke(user, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - if (table_name != nil) - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - if (family != nil) - raise(ArgumentError, "Can't find family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - # invoke cp endpoint to perform access control - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, table_name.to_java_bytes, - fambytes, qualbytes, "".to_java_bytes) - else - user_permission = org.apache.hadoop.hbase.security.access.UserPermission.new( - user.to_java_bytes, "".to_java_bytes) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - begin - protocol.revoke(user_permission) - rescue java.io.IOException => e - if !(e.message.include? "java.lang.NoSuchMethodException") - raise e - end - - # Server has not the new API, try the old one - if (table_name == nil) - raise "Global permissions not supported by HBase Server" - end - - tp = org.apache.hadoop.hbase.security.access.TablePermission.new(table_name.to_java_bytes, fambytes, qualbytes, "".to_java_bytes) - protocol.revoke(user.to_java_bytes, tp) - end - end - - #---------------------------------------------------------------------------------------------- - def user_permission(table_name=nil) - security_available? - - if (table_name != nil) - raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name) - end - - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - protocol = meta_table.coprocessorProxy( - org.apache.hadoop.hbase.security.access.AccessControllerProtocol.java_class, - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - perms = protocol.getUserPermissions(table_name != nil ? table_name.to_java_bytes : nil) - - res = {} - count = 0 - perms.each do |value| - user_name = String.from_java_bytes(value.getUser) - table = (value.getTable != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getTable) : '' - family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' - qualifier = (value.getQualifier != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : '' - - action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions - - if block_given? - yield(user_name, "#{table},#{family},#{qualifier}: #{action.to_s}") - else - res[user_name] ||= {} - res[user_name][family + ":" +qualifier] = action - end - count += 1 - end - - return ((block_given?) ? count : res) - end - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - # Make sure that security classes are available - def security_available?() - begin - org.apache.hadoop.hbase.security.access.AccessControllerProtocol - rescue NameError - raise(ArgumentError, "DISABLED: Security features are not available in this build of HBase") - end - end - - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb deleted file mode 100644 index ed8437ca88..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/hbase/table.rb +++ /dev/null @@ -1,366 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HTable - -module Hbase - class Table - include HBaseConstants - attr_reader :table - @@thread_pool = nil - - def initialize(configuration, table_name, formatter) - if @@thread_pool then - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name.to_java_bytes, @@thread_pool) - else - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name) - @@thread_pool = @table.getPool() - end - end - - #---------------------------------------------------------------------------------------------- - # Put a cell 'value' at specified table/row/column - def put(row, column, value, timestamp = nil) - p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) - family, qualifier = parse_column_name(column) - if timestamp - p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) - else - p.add(family, qualifier, value.to_s.to_java_bytes) - end - @table.put(p) - end - - #---------------------------------------------------------------------------------------------- - # Delete a cell - def delete(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - deleteall(row, column, timestamp) - end - - #---------------------------------------------------------------------------------------------- - # Delete a row - def deleteall(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp, nil) - if column - family, qualifier = parse_column_name(column) - d.deleteColumns(family, qualifier, timestamp) - end - @table.delete(d) - end - - #---------------------------------------------------------------------------------------------- - # Increment a counter atomically - def incr(row, column, value = nil) - value ||= 1 - family, qualifier = parse_column_name(column) - @table.incrementColumnValue(row.to_s.to_java_bytes, family, qualifier, value) - end - - #---------------------------------------------------------------------------------------------- - # Count rows in a table - def count(interval = 1000, caching_rows = 10) - # We can safely set scanner caching with the first key only filter - scan = org.apache.hadoop.hbase.client.Scan.new - scan.cache_blocks = false - scan.caching = caching_rows - scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) - - # Run the scanner - scanner = @table.getScanner(scan) - count = 0 - iter = scanner.iterator - - # Iterate results - while iter.hasNext - row = iter.next - count += 1 - next unless (block_given? && count % interval == 0) - # Allow command modules to visualize counting process - yield(count, - org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) - end - - # Return the counter - return count - end - - #---------------------------------------------------------------------------------------------- - # Get from table - def get(row, *args) - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - maxlength = -1 - - # Normalize args - args = args.first if args.first.kind_of?(Hash) - if args.kind_of?(String) || args.kind_of?(Array) - columns = [ args ].flatten.compact - args = { COLUMNS => columns } - end - - # - # Parse arguments - # - unless args.kind_of?(Hash) - raise ArgumentError, "Failed parse of of #{args.inspect}, #{args.class}" - end - - # Get maxlength parameter if passed - maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] - filter = args.delete(FILTER) if args[FILTER] - - unless args.empty? - columns = args[COLUMN] || args[COLUMNS] - if args[VERSIONS] - vers = args[VERSIONS] - else - vers = 1 - end - if columns - # Normalize types, convert string to an array of strings - columns = [ columns ] if columns.is_a?(String) - - # At this point it is either an array or some unsupported stuff - unless columns.kind_of?(Array) - raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" - end - - # Get each column name and add it to the filter - columns.each do |column| - family, qualifier = parse_column_name(column.to_s) - if qualifier - get.addColumn(family, qualifier) - else - get.addFamily(family) - end - end - - # Additional params - get.setMaxVersions(vers) - get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - else - # May have passed TIMESTAMP and row only; wants all columns from ts. - unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] - raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" - end - - get.setMaxVersions(vers) - # Set the timestamp/timerange - get.setTimeStamp(ts.to_i) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - end - end - - unless filter.class == String - get.setFilter(filter) - else - get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - # Call hbase for the results - result = @table.get(get) - return nil if result.isEmpty - - # Print out results. Result can be Cell or RowResult. - res = {} - result.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - value = to_string(column, kv, maxlength) - - if block_given? - yield(column, value) - else - res[column] = value - end - end - - # If block given, we've yielded all the results, otherwise just return them - return ((block_given?) ? nil : res) - end - - #---------------------------------------------------------------------------------------------- - # Fetches and decodes a counter value from hbase - def get_counter(row, column) - family, qualifier = parse_column_name(column.to_s) - # Format get request - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - get.addColumn(family, qualifier) - get.setMaxVersions(1) - - # Call hbase - result = @table.get(get) - return nil if result.isEmpty - - # Fetch cell value - cell = result.list[0] - org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) - end - - #---------------------------------------------------------------------------------------------- - # Scans whole table or a range of keys and returns rows matching specific criterias - def scan(args = {}) - unless args.kind_of?(Hash) - raise ArgumentError, "Arguments should be a hash. Failed to parse #{args.inspect}, #{args.class}" - end - - limit = args.delete("LIMIT") || -1 - maxlength = args.delete("MAXLENGTH") || -1 - - if args.any? - filter = args["FILTER"] - startrow = args["STARTROW"] || '' - stoprow = args["STOPROW"] - timestamp = args["TIMESTAMP"] - columns = args["COLUMNS"] || args["COLUMN"] || [] - cache_blocks = args["CACHE_BLOCKS"] || true - cache = args["CACHE"] || 0 - versions = args["VERSIONS"] || 1 - timerange = args[TIMERANGE] - raw = args["RAW"] || false - - # Normalize column names - columns = [columns] if columns.class == String - unless columns.kind_of?(Array) - raise ArgumentError.new("COLUMNS must be specified as a String or an Array") - end - - scan = if stoprow - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) - else - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) - end - - columns.each do |c| - family, qualifier = parse_column_name(c.to_s) - if qualifier - scan.addColumn(family, qualifier) - else - scan.addFamily(family) - end - end - - unless filter.class == String - scan.setFilter(filter) - else - scan.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - scan.setTimeStamp(timestamp) if timestamp - scan.setCacheBlocks(cache_blocks) - scan.setCaching(cache) if cache > 0 - scan.setMaxVersions(versions) if versions > 1 - scan.setTimeRange(timerange[0], timerange[1]) if timerange - scan.setRaw(raw) - else - scan = org.apache.hadoop.hbase.client.Scan.new - end - - # Start the scanner - scanner = @table.getScanner(scan) - count = 0 - res = {} - iter = scanner.iterator - - # Iterate results - while iter.hasNext - if limit > 0 && count >= limit - break - end - - row = iter.next - key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) - - row.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - cell = to_string(column, kv, maxlength) - - if block_given? - yield(key, "column=#{column}, #{cell}") - else - res[key] ||= {} - res[key][column] = cell - end - end - - # One more row processed - count += 1 - end - - return ((block_given?) ? count : res) - end - - #---------------------------------------------------------------------------------------- - # Helper methods - - # Returns a list of column names in the table - def get_all_columns - @table.table_descriptor.getFamilies.map do |family| - "#{family.getNameAsString}:" - end - end - - # Checks if current table is one of the 'meta' tables - def is_meta_table? - tn = @table.table_name - org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::META_TABLE_NAME) || org.apache.hadoop.hbase.util.Bytes.equals(tn, org.apache.hadoop.hbase.HConstants::ROOT_TABLE_NAME) - end - - # Returns family and (when has it) qualifier for a column name - def parse_column_name(column) - split = org.apache.hadoop.hbase.KeyValue.parseColumn(column.to_java_bytes) - return split[0], (split.length > 1) ? split[1] : nil - end - - # Make a String of the passed kv - # Intercept cells whose format we know such as the info:regioninfo in .META. - def to_string(column, kv, maxlength = -1) - if is_meta_table? - if column == 'info:regioninfo' or column == 'info:splitA' or column == 'info:splitB' - hri = org.apache.hadoop.hbase.util.Writables.getHRegionInfoOrNull(kv.getValue) - return "timestamp=%d, value=%s" % [kv.getTimestamp, hri.toString] - end - if column == 'info:serverstartcode' - if kv.getValue.length > 0 - str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValue) - else - str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValue) - end - return "timestamp=%d, value=%s" % [kv.getTimestamp, str_val] - end - end - - if kv.isDelete - val = "timestamp=#{kv.getTimestamp}, type=#{org.apache.hadoop.hbase.KeyValue::Type::codeToType(kv.getType)}" - else - val = "timestamp=#{kv.getTimestamp}, value=#{org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getValue)}" - end - (maxlength != -1) ? val[0, maxlength] : val - end - - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb deleted file mode 100644 index 584f70087f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/irb/hirb.rb +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -require 'rbconfig' - -module IRB - WINDOZE = Config::CONFIG['host_os'] =~ /mswin|mingw/ - - # Subclass of IRB so can intercept methods - class HIRB < Irb - def initialize - # This is ugly. Our 'help' method above provokes the following message - # on irb construction: 'irb: warn: can't alias help from irb_help.' - # Below, we reset the output so its pointed at /dev/null during irb - # construction just so this message does not come out after we emit - # the banner. Other attempts at playing with the hash of methods - # down in IRB didn't seem to work. I think the worst thing that can - # happen is the shell exiting because of failed IRB construction with - # no error (though we're not blanking STDERR) - begin - # Map the '/dev/null' according to the runing platform - # Under Windows platform the 'dev/null' is not fully compliant with unix, - # and the 'NUL' object need to be use instead. - devnull = "/dev/null" - devnull = "NUL" if WINDOZE - f = File.open(devnull, "w") - $stdout = f - super - ensure - f.close() - $stdout = STDOUT - end - end - - def output_value - # Suppress output if last_value is 'nil' - # Otherwise, when user types help, get ugly 'nil' - # after all output. - if @context.last_value != nil - super - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb deleted file mode 100644 index 1330b7e834..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell.rb +++ /dev/null @@ -1,315 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Shell commands module -module Shell - @@commands = {} - def self.commands - @@commands - end - - @@command_groups = {} - def self.command_groups - @@command_groups - end - - def self.load_command(name, group) - return if commands[name] - - # Register command in the group - raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] - command_groups[group][:commands] << name - - # Load command - begin - require "shell/commands/#{name}" - klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize - commands[name] = eval("Commands::#{klass_name}") - rescue => e - raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" - end - end - - def self.load_command_group(group, opts) - raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] - - command_groups[group] = { - :commands => [], - :command_names => opts[:commands], - :full_name => opts[:full_name] || group, - :comment => opts[:comment] - } - - opts[:commands].each do |command| - load_command(command, group) - end - end - - #---------------------------------------------------------------------- - class Shell - attr_accessor :hbase - attr_accessor :formatter - - @debug = false - attr_accessor :debug - - def initialize(hbase, formatter) - self.hbase = hbase - self.formatter = formatter - end - - def hbase_admin - @hbase_admin ||= hbase.admin(formatter) - end - - def hbase_table(name) - hbase.table(name, formatter) - end - - def hbase_replication_admin - @hbase_replication_admin ||= hbase.replication_admin(formatter) - end - - def hbase_security_admin - @hbase_security_admin ||= hbase.security_admin(formatter) - end - - def export_commands(where) - ::Shell.commands.keys.each do |cmd| - where.send :instance_eval, <<-EOF - def #{cmd}(*args) - @shell.command('#{cmd}', *args) - puts - end - EOF - end - end - - def command_instance(command) - ::Shell.commands[command.to_s].new(self) - end - - def command(command, *args) - command_instance(command).command_safe(self.debug, *args) - end - - def print_banner - puts "HBase Shell; enter 'help' for list of supported commands." - puts 'Type "exit" to leave the HBase Shell' - print 'Version ' - command('version') - puts - end - - def help_multi_command(command) - puts "Command: #{command}" - puts command_instance(command).help - puts - return nil - end - - def help_command(command) - puts command_instance(command).help - return nil - end - - def help_group(group_name) - group = ::Shell.command_groups[group_name.to_s] - group[:commands].sort.each { |cmd| help_multi_command(cmd) } - if group[:comment] - puts '-' * 80 - puts - puts group[:comment] - puts - end - return nil - end - - def help(command = nil) - if command - return help_command(command) if ::Shell.commands[command.to_s] - return help_group(command) if ::Shell.command_groups[command.to_s] - puts "ERROR: Invalid command or command group name: #{command}" - puts - end - - puts help_header - puts - puts 'COMMAND GROUPS:' - ::Shell.command_groups.each do |name, group| - puts " Group name: " + name - puts " Commands: " + group[:command_names].sort.join(', ') - puts - end - unless command - puts 'SHELL USAGE:' - help_footer - end - return nil - end - - def help_header - return "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + "\n" + - "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" + - "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." - end - - def help_footer - puts <<-HERE -Quote all names in HBase Shell such as table and column names. Commas delimit -command parameters. Type after entering a command to run it. -Dictionaries of configuration used in the creation and alteration of tables are -Ruby Hashes. They look like this: - - {'key1' => 'value1', 'key2' => 'value2', ...} - -and are opened and closed with curley-braces. Key/values are delimited by the -'=>' character combination. Usually keys are predefined constants such as -NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type -'Object.constants' to see a (messy) list of all constants in the environment. - -If you are using binary keys or values and need to enter them in the shell, use -double-quote'd hexadecimal representation. For example: - - hbase> get 't1', "key\\x03\\x3f\\xcd" - hbase> get 't1', "key\\003\\023\\011" - hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" - -The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. -For more on the HBase Shell, see http://hbase.apache.org/docs/current/book.html - HERE - end - end -end - -# Load commands base class -require 'shell/commands' - -# Load all commands -Shell.load_command_group( - 'general', - :full_name => 'GENERAL HBASE SHELL COMMANDS', - :commands => %w[ - status - version - whoami - ] -) - -Shell.load_command_group( - 'ddl', - :full_name => 'TABLES MANAGEMENT COMMANDS', - :commands => %w[ - alter - create - describe - disable - disable_all - is_disabled - drop - drop_all - enable - enable_all - is_enabled - exists - list - show_filters - alter_status - alter_async - ] -) - -Shell.load_command_group( - 'dml', - :full_name => 'DATA MANIPULATION COMMANDS', - :commands => %w[ - count - delete - deleteall - get - get_counter - incr - put - scan - truncate - ] -) - -Shell.load_command_group( - 'tools', - :full_name => 'HBASE SURGERY TOOLS', - :comment => "WARNING: Above commands are for 'experts'-only as misuse can damage an install", - :commands => %w[ - assign - balancer - balance_switch - close_region - compact - flush - major_compact - move - split - unassign - zk_dump - hlog_roll - ] -) - -Shell.load_command_group( - 'replication', - :full_name => 'CLUSTER REPLICATION TOOLS', - :comment => "In order to use these tools, hbase.replication must be true. enabling/disabling is currently unsupported", - :commands => %w[ - add_peer - remove_peer - list_peers - enable_peer - disable_peer - start_replication - stop_replication - list_replicated_tables - ] -) - -Shell.load_command_group( - 'snapshot', - :full_name => 'CLUSTER SNAPSHOT TOOLS', - :commands => %w[ - snapshot - clone_snapshot - restore_snapshot - delete_snapshot - list_snapshots - ] -) - -Shell.load_command_group( - 'security', - :full_name => 'SECURITY TOOLS', - :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", - :commands => %w[ - grant - revoke - user_permission - ] -) - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb deleted file mode 100644 index af6df33440..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands.rb +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2009 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Command - attr_accessor :shell - - def initialize(shell) - self.shell = shell - end - - def command_safe(debug, *args) - translate_hbase_exceptions(*args) { command(*args) } - rescue => e - puts - puts "ERROR: #{e}" - puts "Backtrace: #{e.backtrace.join("\n ")}" if debug - puts - puts "Here is some help for this command:" - puts help - puts - ensure - return nil - end - - def admin - shell.hbase_admin - end - - def table(name) - shell.hbase_table(name) - end - - def replication_admin - shell.hbase_replication_admin - end - - def security_admin - shell.hbase_security_admin - end - - #---------------------------------------------------------------------- - - def formatter - shell.formatter - end - - def format_simple_command - now = Time.now - yield - formatter.header - formatter.footer(now) - end - - def translate_hbase_exceptions(*args) - yield - rescue org.apache.hadoop.hbase.TableNotFoundException - raise "Unknown table #{args.first}!" - rescue org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException - valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } - raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" - rescue org.apache.hadoop.hbase.TableExistsException - raise "Table already exists: #{args.first}!" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb deleted file mode 100644 index 7669fb7c52..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/add_peer.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AddPeer< Command - def help - return <<-EOF -Add a peer cluster to replicate to, the id must be a short and -the cluster key is composed like this: -hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent -This gives a full path for HBase to connect to another cluster. -Examples: - - hbase> add_peer '1', "server1.cie.com:2181:/hbase" - hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" -EOF - end - - def command(id, cluster_key) - format_simple_command do - replication_admin.add_peer(id, cluster_key) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb deleted file mode 100644 index 3be8f0a427..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter.rb +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Alter < Command - def help - return <<-EOF -Alter column family schema; pass table name and a dictionary -specifying new column family schema. Dictionaries are described -on the main help command output. Dictionary must include name -of column family to alter. For example, - -To change or add the 'f1' column family in table 't1' from defaults -to instead keep a maximum of 5 cell VERSIONS, do: - - hbase> alter 't1', NAME => 'f1', VERSIONS => 5 - -To delete the 'f1' column family in table 't1', do: - - hbase> alter 't1', NAME => 'f1', METHOD => 'delete' - -or a shorter version: - - hbase> alter 't1', 'delete' => 'f1' - -You can also change the column family config by set attribute CONFIG like this: - hbase> alter 'test', NAME=>'f', CONFIG => {'hbase.hstore.compaction.min' => '5'} - -You can also change table-scope attributes like MAX_FILESIZE -MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. - -For example, to change the max size of a family to 128MB, do: - - hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' - -You can also change the table-scope by set attribute CONFIG like this: - hbase> alter 'test', METHOD=>'table_att', CONFIG => {'hbase.hstore.compaction.min' => '5'} - -You can add a table coprocessor by setting a table coprocessor attribute: - - hbase> alter 't1', METHOD => 'table_att', - 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' - -Since you can have multiple coprocessors configured for a table, a -sequence number will be automatically appended to the attribute name -to uniquely identify it. - -The coprocessor attribute must match the pattern below in order for -the framework to understand how to load the coprocessor classes: - - [coprocessor jar file location] | class name | [priority] | [arguments] - -You can also remove a table-scope attribute: - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' - -There could be more than one alteration in one command: - - hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, true, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb deleted file mode 100644 index 01dfd99465..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_async.rb +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterAsync < Command - def help - return <<-EOF -Alter column family schema, does not wait for all regions to receive the -schema changes. Pass table name and a dictionary specifying new column -family schema. Dictionaries are described on the main help command output. -Dictionary must include name of column family to alter. For example, - -To change or add the 'f1' column family in table 't1' from defaults -to instead keep a maximum of 5 cell VERSIONS, do: - - hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 - -To delete the 'f1' column family in table 't1', do: - - hbase> alter_async 't1', NAME => 'f1', METHOD => 'delete' - -or a shorter version: - - hbase> alter_async 't1', 'delete' => 'f1' - -You can also change table-scope attributes like MAX_FILESIZE -MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. - -For example, to change the max size of a family to 128MB, do: - - hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' - -There could be more than one alteration in one command: - - hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} - -To check if all the regions have been updated, use alter_status -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, false, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb deleted file mode 100644 index cc79e1108b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/alter_status.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterStatus < Command - def help - return <<-EOF -Get the status of the alter command. Indicates the number of regions of the -table that have received the updated schema -Pass table name. - -hbase> alter_status 't1' -EOF - end - def command(table) - admin.alter_status(table) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb deleted file mode 100644 index 4c83d3cf20..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/assign.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Assign < Command - def help - return <<-EOF -Assign a region.Use with caution.If region already assigned, -this command will just go ahead and reassign -the region anyways. For experts only. -EOF - end - - def command(region_name) - format_simple_command do - admin.assign(region_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb deleted file mode 100644 index 0eac7654a8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balance_switch.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class BalanceSwitch < Command - def help - return <<-EOF -Enable/Disable balancer. Returns previous balancer state. -Examples: - - hbase> balance_switch true - hbase> balance_switch false -EOF - end - - def command(enableDisable) - format_simple_command do - formatter.row([ - admin.balance_switch(enableDisable)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb deleted file mode 100644 index 05ecd3a2bc..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/balancer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Balancer < Command - def help - return <<-EOF -Trigger the cluster balancer. Returns true if balancer ran and was able to -tell the region servers to unassign all the regions to balance (the re-assignment itself is async). -Otherwise false (Will not run if regions in transition). -EOF - end - - def command() - format_simple_command do - formatter.row([ - admin.balancer()? "true": "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb deleted file mode 100644 index 43d240f12f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/clone_snapshot.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloneSnapshot < Command - def help - return <<-EOF -Create a new table by cloning the snapshot content. -There're no copies of data involved. -And writing on the newly created table will not influence the snapshot data. - -Examples: - hbase> clone_snapshot 'snapshotName', 'tableName' -EOF - end - - def command(snapshot_name, table) - format_simple_command do - admin.clone_snapshot(snapshot_name, table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb deleted file mode 100644 index f1010ac6ca..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/close_region.rb +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloseRegion < Command - def help - return <<-EOF -Close a single region. Ask the master to close a region out on the cluster -or if 'SERVER_NAME' is supplied, ask the designated hosting regionserver to -close the region directly. Closing a region, the master expects 'REGIONNAME' -to be a fully qualified region name. When asking the hosting regionserver to -directly close a region, you pass the regions' encoded name only. A region -name looks like this: - - TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. - -The trailing period is part of the regionserver name. A region's encoded name -is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 -(without the period). A 'SERVER_NAME' is its host, port plus startcode. For -example: host187.example.com,60020,1289493121758 (find servername in master ui -or when you do detailed status in shell). This command will end up running -close on the region hosting regionserver. The close is done without the -master's involvement (It will not know of the close). Once closed, region will -stay closed. Use assign to reopen/reassign. Use unassign or move to assign -the region elsewhere on cluster. Use with caution. For experts only. -Examples: - - hbase> close_region 'REGIONNAME' - hbase> close_region 'REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(region_name, server = nil) - format_simple_command do - admin.close_region(region_name, server) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb deleted file mode 100644 index fa0b82eef8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/compact.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Compact < Command - def help - return <<-EOF - Compact all regions in passed table or pass a region row - to compact an individual region. You can also compact a single column - family within a region. - Examples: - Compact all regions in a table: - hbase> compact 't1' - Compact an entire region: - hbase> compact 'r1' - Compact only a column family within a region: - hbase> compact 'r1', 'c1' - Compact a column family within a table: - hbase> compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb deleted file mode 100644 index 6596441c1c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/count.rb +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Count < Command - def help - return <<-EOF -Count the number of rows in a table. This operation may take a LONG -time (Run '$HADOOP_HOME/bin/hadoop jar hbase.jar rowcount' to run a -counting mapreduce job). Current count is shown every 1000 rows by -default. Count interval may be optionally specified. Scan caching -is enabled on count scans by default. Default cache size is 10 rows. -If your rows are small in size, you may want to increase this -parameter. Examples: - - hbase> count 't1' - hbase> count 't1', INTERVAL => 100000 - hbase> count 't1', CACHE => 1000 - hbase> count 't1', INTERVAL => 10, CACHE => 1000 -EOF - end - - def command(table, params = {}) - # If the second parameter is an integer, then it is the old command syntax - params = { 'INTERVAL' => params } if params.kind_of?(Fixnum) - - # Merge params with defaults - params = { - 'INTERVAL' => 1000, - 'CACHE' => 10 - }.merge(params) - - # Call the counter method - now = Time.now - formatter.header - count = table(table).count(params['INTERVAL'].to_i, params['CACHE'].to_i) do |cnt, row| - formatter.row([ "Current count: #{cnt}, row: #{row}" ]) - end - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb deleted file mode 100644 index 14c1b0ff4f..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/create.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Create < Command - def help - return <<-EOF -Create table; pass table name, a dictionary of specifications per -column family, and optionally a dictionary of table configuration. -Dictionaries are described below in the GENERAL NOTES section. -Examples: - - hbase> create 't1', {NAME => 'f1', VERSIONS => 5} - hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} - hbase> # The above in shorthand would be the following: - hbase> create 't1', 'f1', 'f2', 'f3' - hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} - hbase> create 't1', 'f1', {SPLITS => ['10', '20', '30', '40']} - hbase> create 't1', 'f1', {SPLITS_FILE => 'splits.txt'} - hbase> # Optionally pre-split the table into NUMREGIONS, using - hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) - hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} -EOF - end - - def command(table, *args) - format_simple_command do - admin.create(table, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb deleted file mode 100644 index 12bc405933..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Delete < Command - def help - return <<-EOF -Put a delete cell value at specified table/row/column and optionally -timestamp coordinates. Deletes must match the deleted cell's -coordinates exactly. When scanning, a delete cell suppresses older -versions. To delete a cell from 't1' at row 'r1' under column 'c1' -marked with the time 'ts1', do: - - hbase> delete 't1', 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table(table).delete(row, column, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb deleted file mode 100644 index b8c3791a54..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/delete_snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DeleteSnapshot < Command - def help - return <<-EOF -Delete a specified snapshot. Examples: - - hbase> delete_snapshot 'snapshotName', -EOF - end - - def command(snapshot_name) - format_simple_command do - admin.delete_snapshot(snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb deleted file mode 100644 index 5731b606b4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/deleteall.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Deleteall < Command - def help - return <<-EOF -Delete all cells in a given row; pass a table name, row, and optionally -a column and timestamp. Examples: - - hbase> deleteall 't1', 'r1' - hbase> deleteall 't1', 'r1', 'c1' - hbase> deleteall 't1', 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table(table).deleteall(row, column, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb deleted file mode 100644 index 0f355074cd..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/describe.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Describe < Command - def help - return <<-EOF -Describe the named table. For example: - hbase> describe 't1' -EOF - end - - def command(table) - now = Time.now - - desc = admin.describe(table) - - formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) - formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb deleted file mode 100644 index 34c5f9ca7d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Disable < Command - def help - return <<-EOF -Start disable of named table: e.g. "hbase> disable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.disable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb deleted file mode 100644 index b7c9cf5fdd..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DisableAll < Command - def help - return <<-EOF -Disable all of tables matching the given regex: - -hbase> disable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.disable_all(regex) - puts "#{count - failed.size} tables successfully disabled" - puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb deleted file mode 100644 index da9941a957..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/disable_peer.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DisablePeer< Command - def help - return <<-EOF -Stops the replication stream to the specified cluster, but still -keeps track of new edits to replicate. - -Examples: - - hbase> disable_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.disable_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb deleted file mode 100644 index 40831d2fa0..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Drop < Command - def help - return <<-EOF -Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.drop(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb deleted file mode 100644 index dcbefa3234..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/drop_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DropAll < Command - def help - return <<-EOF -Drop all of the tables matching the given regex: - -hbase> drop_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.drop_all(regex) - puts "#{count - failed.size} tables successfully dropped" - puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb deleted file mode 100644 index a0dc340f5d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Enable < Command - def help - return <<-EOF -Start enable of named table: e.g. "hbase> enable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.enable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb deleted file mode 100644 index ee016373f4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_all.rb +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class EnableAll < Command - def help - return <<-EOF -Enable all of the tables matching the given regex: - -hbase> enable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.enable_all(regex) - puts "#{count - failed.size} tables successfully enabled" - puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb deleted file mode 100644 index de459092b5..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/enable_peer.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class EnablePeer< Command - def help - return <<-EOF -Restarts the replication to the specified peer cluster, -continuing from where it was disabled. - -Examples: - - hbase> enable_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.enable_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb deleted file mode 100644 index f35f197238..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/exists.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Exists < Command - def help - return <<-EOF -Does the named table exist? e.g. "hbase> exists 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - "Table #{table} " + (admin.exists?(table.to_s) ? "does exist" : "does not exist") - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb deleted file mode 100644 index ba597662c5..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/flush.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Flush < Command - def help - return <<-EOF -Flush all regions in passed table or pass a region row to -flush an individual region. For example: - - hbase> flush 'TABLENAME' - hbase> flush 'REGIONNAME' -EOF - end - - def command(table_or_region_name) - format_simple_command do - admin.flush(table_or_region_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb deleted file mode 100644 index 8456d58a5c..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get.rb +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Get < Command - def help - return <<-EOF -Get row or cell contents; pass table name, row, and optionally -a dictionary of column(s), timestamp, timerange and versions. Examples: - - hbase> get 't1', 'r1' - hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} - hbase> get 't1', 'r1', {COLUMN => 'c1'} - hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} - hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} - hbase> get 't1', 'r1', 'c1' - hbase> get 't1', 'r1', 'c1', 'c2' - hbase> get 't1', 'r1', ['c1', 'c2'] -EOF - end - - def command(table, row, *args) - now = Time.now - formatter.header(["COLUMN", "CELL"]) - - table(table).get(row, *args) do |column, value| - formatter.row([ column, value ]) - end - - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb deleted file mode 100644 index 3cbe2268d9..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/get_counter.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class GetCounter < Command - def help - return <<-EOF -Return a counter cell value at specified table/row/column coordinates. -A cell cell should be managed with atomic increment function oh HBase -and the data should be binary encoded. Example: - - hbase> get_counter 't1', 'r1', 'c1' -EOF - end - - def command(table, row, column, value = nil) - if cnt = table(table).get_counter(row, column) - puts "COUNTER VALUE = #{cnt}" - else - puts "No counter found at specified coordinates" - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb deleted file mode 100644 index 7bdc95943b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/grant.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Grant < Command - def help - return <<-EOF -Grant users specific rights. -Syntax : grant [ [ []] - -permissions is either zero or more letters from the set "RWXCA". -READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') - -For example: - - hbase> grant 'bobsmith', 'RWXCA' - hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' -EOF - end - - def command(user, rights, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.grant(user, rights, table_name, family, qualifier) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb deleted file mode 100644 index 02c7c59282..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/hlog_roll.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -module Shell - module Commands - class HlogRoll < Command - def help - return <<-EOF -Roll the log writer. That is, start writing log messages to a new file. -The name of the regionserver should be given as the parameter. A -'server_name' is the host, port plus startcode of a regionserver. For -example: host187.example.com,60020,1289493121758 (find servername in -master ui or when you do detailed status in shell) -EOF - end - - def command(server_name) - format_simple_command do - admin.hlog_roll(server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb deleted file mode 100644 index 38a2fc5179..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/incr.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Incr < Command - def help - return <<-EOF -Increments a cell 'value' at specified table/row/column coordinates. -To increment a cell value in table 't1' at row 'r1' under column -'c1' by 1 (can be omitted) or 10 do: - - hbase> incr 't1', 'r1', 'c1' - hbase> incr 't1', 'r1', 'c1', 1 - hbase> incr 't1', 'r1', 'c1', 10 -EOF - end - - def command(table, row, column, value = nil) - cnt = table(table).incr(row, column, value) - puts "COUNTER VALUE = #{cnt}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb deleted file mode 100644 index 9d3c7ee747..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_disabled.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsDisabled < Command - def help - return <<-EOF -Is named table disabled?: e.g. "hbase> is_disabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.disabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb deleted file mode 100644 index 96b2b152c2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/is_enabled.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsEnabled < Command - def help - return <<-EOF -Is named table enabled?: e.g. "hbase> is_enabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.enabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb deleted file mode 100644 index 592fb5eec2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class List < Command - def help - return <<-EOF -List all tables in hbase. Optional regular expression parameter could -be used to filter the output. Examples: - - hbase> list - hbase> list 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "TABLE" ]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - list.each do |table| - formatter.row([ table ]) - end - - formatter.footer(now, list.size) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb deleted file mode 100644 index 2f57592e58..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_peers.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListPeers< Command - def help - return <<-EOF -List all replication peer clusters. - - hbase> list_peers -EOF - end - - def command() - now = Time.now - peers = replication_admin.list_peers - - formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"]) - - peers.entrySet().each do |e| - state = replication_admin.get_peer_state(e.key) - formatter.row([ e.key, e.value, state ]) - end - - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb deleted file mode 100644 index b1494b8d3a..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_replicated_tables.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListReplicatedTables< Command - def help - return <<-EOF -List all the tables and column families replicated from this cluster - - hbase> list_replicated_tables - hbase> list_replicated_tables 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - - formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ]) - list = replication_admin.list_replicated_tables - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))} - list.each do |e| - if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL - replicateType = "GLOBAL" - else - replicateType = "unknown" - end - formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32]) - end - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb deleted file mode 100644 index 95135185c2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/list_snapshots.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'time' - -module Shell - module Commands - class ListSnapshots < Command - def help - return <<-EOF -List all snapshots taken (by printing the names and relative information). -Optional regular expression parameter could be used to filter the output -by snapshot name. - -Examples: - hbase> list_snapshots - hbase> list_snapshots 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_snapshot.select {|s| regex.match(s.getName)} - list.each do |snapshot| - creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s - formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) - end - - formatter.footer(now, list.size) - return list.map { |s| s.getName() } - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb deleted file mode 100644 index 325344b526..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/major_compact.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class MajorCompact < Command - def help - return <<-EOF - Run major compaction on passed table or pass a region row - to major compact an individual region. To compact a single - column family within a region specify the region name - followed by the column family name. - Examples: - Compact all regions in a table: - hbase> major_compact 't1' - Compact an entire region: - hbase> major_compact 'r1' - Compact a single column family within a region: - hbase> major_compact 'r1', 'c1' - Compact a single column family within a table: - hbase> major_compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.major_compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb deleted file mode 100644 index 0e3db8fe47..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/move.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Move < Command - def help - return <<-EOF -Move a region. Optionally specify target regionserver else we choose one -at random. NOTE: You pass the encoded region name, not the region name so -this command is a little different to the others. The encoded region name -is the hash suffix on region names: e.g. if the region name were -TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then -the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 -A server name is its host, port plus startcode. For example: -host187.example.com,60020,1289493121758 -Examples: - - hbase> move 'ENCODED_REGIONNAME' - hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(encoded_region_name, server_name = nil) - format_simple_command do - admin.move(encoded_region_name, server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb deleted file mode 100644 index dde0433708..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/put.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Put < Command - def help - return <<-EOF -Put a cell 'value' at specified table/row/column and optionally -timestamp coordinates. To put a cell value into table 't1' at -row 'r1' under column 'c1' marked with the time 'ts1', do: - - hbase> put 't1', 'r1', 'c1', 'value', ts1 -EOF - end - - def command(table, row, column, value, timestamp = nil) - format_simple_command do - table(table).put(row, column, value, timestamp) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb deleted file mode 100644 index 034434aa5d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/remove_peer.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RemovePeer< Command - def help - return <<-EOF -Stops the specified replication stream and deletes all the meta -information kept about it. Examples: - - hbase> remove_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.remove_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb deleted file mode 100644 index 4d531711bc..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/restore_snapshot.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RestoreSnapshot < Command - def help - return <<-EOF -Restore a specified snapshot. -The restore will replace the content of the original table, -bringing back the content to the snapshot state. -The table must be disabled. - -Examples: - hbase> restore_snapshot 'snapshotName' -EOF - end - - def command(snapshot_name) - format_simple_command do - admin.restore_snapshot(snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb deleted file mode 100644 index 721519e7d6..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/revoke.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Revoke < Command - def help - return <<-EOF -Revoke a user's access rights. -Syntax : revoke [
[ []] -For example: - - hbase> revoke 'bobsmith' - hbase> revoke 'bobsmith', 't1', 'f1', 'col1' -EOF - end - - def command(user, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.revoke(user, table_name, family, qualifier) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb deleted file mode 100644 index 518e628d67..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/scan.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Scan < Command - def help - return <<-EOF -Scan a table; pass table name and optionally a dictionary of scanner -specifications. Scanner specifications may include one or more of: -TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, -or COLUMNS, CACHE - -If no columns are specified, all columns will be scanned. -To scan all members of a column family, leave the qualifier empty as in -'col_family:'. - -The filter can be specified in two ways: -1. Using a filterString - more information on this is available in the -Filter Language document attached to the HBASE-4176 JIRA -2. Using the entire package name of the filter. - -Some examples: - - hbase> scan '.META.' - hbase> scan '.META.', {COLUMNS => 'info:regioninfo'} - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} - hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} - hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} - hbase> scan 't1', {FILTER => org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} - -For experts, there is an additional option -- CACHE_BLOCKS -- which -switches block caching for the scanner on (true) or off (false). By -default it is enabled. Examples: - - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} - -Also for experts, there is an advanced option -- RAW -- which instructs the -scanner to return all cells (including delete markers and uncollected deleted -cells). This option cannot be combined with requesting specific COLUMNS. -Disabled by default. Example: - - hbase> scan 't1', {RAW => true, VERSIONS => 10} -EOF - end - - def command(table, args = {}) - now = Time.now - formatter.header(["ROW", "COLUMN+CELL"]) - - count = table(table).scan(args) do |row, cells| - formatter.row([ row, cells ]) - end - - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb deleted file mode 100644 index 1581db5b37..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/show_filters.rb +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -java_import org.apache.hadoop.hbase.filter.ParseFilter - -module Shell - module Commands - class ShowFilters < Command - def help - return <<-EOF -Show all the filters in hbase. Example: - hbase> show_filters - - Documentation on filters mentioned below can be found at: https://our.intern.facebook.com/intern/wiki/index.php/HBase/Filter_Language - ColumnPrefixFilter - TimestampsFilter - PageFilter - ..... - KeyOnlyFilter -EOF - end - - def command( ) - now = Time.now - formatter.row(["Documentation on filters mentioned below can " + - "be found at: https://our.intern.facebook.com/intern/" + - "wiki/index.php/HBase/Filter_Language"]) - - parseFilter = ParseFilter.new - supportedFilters = parseFilter.getSupportedFilters - - supportedFilters.each do |filter| - formatter.row([filter]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb deleted file mode 100644 index 1c4ecfec0e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Snapshot < Command - def help - return <<-EOF -Take a snapshot of specified table. Examples: - - hbase> snapshot 'sourceTable', 'snapshotName' -EOF - end - - def command(table, snapshot_name) - format_simple_command do - admin.snapshot(table, snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb deleted file mode 100644 index 47e21f86c2..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/split.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Split < Command - def help - return <<-EOF -Split entire table or pass a region to split individual region. With the -second parameter, you can specify an explicit split key for the region. -Examples: - split 'tableName' - split 'regionName' # format: 'tableName,startKey,id' - split 'tableName', 'splitKey' - split 'regionName', 'splitKey' -EOF - end - - def command(table_or_region_name, split_point = nil) - format_simple_command do - admin.split(table_or_region_name, split_point) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb deleted file mode 100644 index 5d1cd1b5ee..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/start_replication.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class StartReplication < Command - def help - return <<-EOF -Restarts all the replication features. The state in which each -stream starts in is undetermined. -WARNING: -start/stop replication is only meant to be used in critical load situations. -Examples: - - hbase> start_replication -EOF - end - - def command - format_simple_command do - replication_admin.start_replication - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb deleted file mode 100644 index 4b22acb4ab..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/status.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Status < Command - def help - return <<-EOF -Show cluster status. Can be 'summary', 'simple', or 'detailed'. The -default is 'summary'. Examples: - - hbase> status - hbase> status 'simple' - hbase> status 'summary' - hbase> status 'detailed' -EOF - end - - def command(format = 'summary') - admin.status(format) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb deleted file mode 100644 index f5074d73a8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/stop_replication.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class StopReplication < Command - def help - return <<-EOF -Stops all the replication features. The state in which each -stream stops in is undetermined. -WARNING: -start/stop replication is only meant to be used in critical load situations. -Examples: - - hbase> stop_replication -EOF - end - - def command - format_simple_command do - replication_admin.stop_replication - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb deleted file mode 100644 index a24e167b98..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/truncate.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Truncate < Command - def help - return <<-EOF - Disables, drops and recreates the specified table. -EOF - end - - def command(table) - format_simple_command do - puts "Truncating '#{table}' table (it may take a while):" - admin.truncate(table) { |log| puts " - #{log}" } - end - end - - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb deleted file mode 100644 index 75242baf4b..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/unassign.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Unassign < Command - def help - return <<-EOF -Unassign a region. Unassign will close region in current location and then -reopen it again. Pass 'true' to force the unassignment ('force' will clear -all in-memory state in master before the reassign. If results in -double assignment use hbck -fix to resolve. To be used by experts). -Use with caution. For expert use only. Examples: - - hbase> unassign 'REGIONNAME' - hbase> unassign 'REGIONNAME', true -EOF - end - - def command(region_name, force = 'false') - format_simple_command do - admin.unassign(region_name, force) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb deleted file mode 100644 index ad4a7b029e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/user_permission.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class UserPermission < Command - def help - return <<-EOF -Show all permissions for the particular user. -Syntax : user_permission
-For example: - - hbase> user_permission - hbase> user_permission 'table1' -EOF - end - - def command(table=nil) - #format_simple_command do - #admin.user_permission(table) - now = Time.now - formatter.header(["User", "Table,Family,Qualifier:Permission"]) - - count = security_admin.user_permission(table) do |user, permission| - formatter.row([ user, permission]) - end - - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb deleted file mode 100644 index 372b0dc341..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/version.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Version < Command - def help - return <<-EOF -Output this HBase version -EOF - end - - def command - # Output version. - puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb deleted file mode 100644 index 040ad7e7ef..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/whoami.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Whoami < Command - def help - return <<-EOF -Show the current hbase user. -Syntax : whoami -For example: - - hbase> whoami -EOF - end - - def command() - puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb deleted file mode 100644 index bb23962df8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/commands/zk_dump.rb +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ZkDump < Command - def help - return <<-EOF -Dump status of HBase cluster as seen by ZooKeeper. -EOF - end - - def command - puts admin.zk_dump - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb b/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb deleted file mode 100644 index ea17882a6e..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/ruby/shell/formatter.rb +++ /dev/null @@ -1,171 +0,0 @@ -# -# Copyright 2010 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Results formatter -module Shell - module Formatter - # Base abstract class for results formatting. - class Base - attr_reader :row_count - - def is_valid_io?(obj) - obj.instance_of?(IO) || obj == Kernel - end - - def refresh_width() - if $stdout.tty? - @max_width = Java::jline.Terminal.getTerminal().getTerminalWidth() - else - @max_width = 0 - end - end - - # Takes an output stream and a print width. - def initialize(opts = {}) - options = { - :output_stream => Kernel, - }.merge(opts) - - @out = options[:output_stream] - refresh_width - @row_count = 0 - - # raise an error if the stream is not valid - raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) - end - - def header(args = [], widths = []) - refresh_width - row(args, false, widths) if args.length > 0 - @row_count = 0 - end - - # Output a row. - # Inset is whether or not to offset row by a space. - def row(args = [], inset = true, widths = []) - # Print out nothing - return if !args || args.empty? - - # Print a string - if args.is_a?(String) - output(args) - @out.puts - return - end - - # TODO: Look at the type. Is it RowResult? - if args.length == 1 - splits = split(@max_width, dump(args[0])) - for l in splits - output(@max_width, l) - @out.puts - end - elsif args.length == 2 - if @max_width == 0 - col1width = col2width = 0 - else - col1width = (not widths or widths.length == 0) ? @max_width / 4 : @max_width * widths[0] / 100 - col2width = (not widths or widths.length < 2) ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 - end - splits1 = split(col1width, dump(args[0])) - splits2 = split(col2width, dump(args[1])) - biggest = (splits2.length > splits1.length)? splits2.length: splits1.length - index = 0 - while index < biggest - # Inset by one space if inset is set. - @out.print(" ") if inset - output(col1width, splits1[index]) - # Add extra space so second column lines up w/ second column output - @out.print(" ") unless inset - @out.print(" ") - output(col2width, splits2[index]) - index += 1 - @out.puts - end - else - # Print a space to set off multi-column rows - print ' ' - first = true - for e in args - @out.print " " unless first - first = false - @out.print e - end - puts - end - @row_count += 1 - end - - def split(width, str) - if width == 0 - return [str] - end - result = [] - index = 0 - while index < str.length do - result << str.slice(index, width) - index += width - end - result - end - - def dump(str) - return if str.instance_of?(Fixnum) - # Remove double-quotes added by 'dump'. - return str - end - - def output(str) - output(@max_width, str) - end - - def output(width, str) - if str == nil - str = '' - end - if not width or width == str.length - @out.print(str) - else - @out.printf('%-*s', width, str) - end - end - - def footer(start_time = nil, row_count = nil) - return unless start_time - row_count ||= @row_count - # Only output elapsed time and row count if startTime passed - @out.puts("%d row(s) in %.4f seconds" % [row_count, Time.now - start_time]) - end - end - - - class Console < Base - end - - class XHTMLFormatter < Base - # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html - # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html - end - - class JSON < Base - end - end -end - diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh deleted file mode 100755 index 6f3cd9085d..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/start-hbase.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -# Start hadoop hbase daemons. -# Run this on master node. -usage="Usage: start-hbase.sh" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` - - -if [ "$distMode" == 'false' ] -then - "$bin"/hbase-daemon.sh start master -else - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" start regionserver - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh deleted file mode 100755 index e9749fee28..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/stop-hbase.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -# Stop hadoop hbase daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# variables needed for stop command -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid - -echo -n stopping hbase -echo "`date` Stopping hbase (via master)" >> $loglog - -nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - master stop "$@" > "$logout" 2>&1 < /dev/null & - -while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; -done -# Add a CR after we're done w/ dots. -echo - -# distributed == false means that the HMaster will kill ZK when it exits -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'true' ] -then - # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message - # stop any backup masters - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper -fi diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch b/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch deleted file mode 100644 index 94906ceab4..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hbase_script.patch +++ /dev/null @@ -1,28 +0,0 @@ ---- hbase.orig 2013-10-23 18:14:51.867728302 -0400 -+++ hbase 2013-10-23 20:11:57.564530457 -0400 -@@ -137,7 +137,24 @@ - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then -- ${MVN} -f "${HBASE_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ # ------------------- -+ # Titan build support -+ # ------------------- -+ # Check for Maven command line options file. -+ # If titan-hbase has already been built, then this file will contain options -+ # to target the version of Hadoop used during compilation. This is -+ # necessary when the user compiles titan-hbase with a non-default Hadoop -+ # version (e.g. 2.2.0). If we omitted this check, then we'd try to start -+ # HBase with the default Hadoop version (currentl 1.0.4), and it would break. -+ MVN_OPTS= -+ optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" -+ unset IFS -+ if [ -f "${optf}" ] -+ then -+ MVN_OPTS=`cat "${optf}"` -+ fi -+ ${MVN} -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` - } diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch b/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch deleted file mode 100644 index 0d4dc74cae..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/titan_customizations_to_hirb.rb.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/titan-hbase/bin/hirb.rb b/titan-hbase/bin/hirb.rb -index 32a51b3..c792515 100755 ---- a/titan-hbase/bin/hirb.rb -+++ b/titan-hbase/bin/hirb.rb -@@ -37,12 +37,9 @@ include Java - # Some goodies for hirb. Should these be left up to the user's discretion? - require 'irb/completion' - --# Add the $HBASE_HOME/lib/ruby OR $HBASE_HOME/src/main/ruby/lib directory --# to the ruby load path so I can load up my HBase ruby modules --if File.exists?(File.join(File.dirname(__FILE__), "..", "lib", "ruby", "hbase.rb")) -- $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib", "ruby") --else -- $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "src", "main", "ruby") -+# Add HBase to the ruby load path -+if File.exists?(File.join(File.dirname(__FILE__), "ruby", "hbase.rb")) -+ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "ruby") - end - - # diff --git a/titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh b/titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh deleted file mode 100755 index 89a214e5a8..0000000000 --- a/titan-hbase-parent/titan-hbase-094/src/test/bin/zookeepers.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -if [ "$HBASE_MANAGES_ZK" = "" ]; then - HBASE_MANAGES_ZK=true -fi - -if [ "$HBASE_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-hbase-parent/titan-hbase-096/pom.xml b/titan-hbase-parent/titan-hbase-096/pom.xml deleted file mode 100644 index 2ea2f7a92d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/pom.xml +++ /dev/null @@ -1,210 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-hbase-parent - 1.0.1-SNAPSHOT - ../pom.xml - - titan-hbase-096 - Titan-HBase: 0.96 Compatibility Shim - http://thinkaurelius.github.com/titan/ - - - 1.6.8 - ${project.parent.parent.basedir} - - - - - ${project.groupId} - titan-hbase-core - ${project.version} - - - commons-net - commons-net - - - - - org.apache.hbase - hbase-client - ${hbase096.version} - true - - - avro - org.apache.avro - - - jruby-complete - org.jruby - - - asm - asm - - - - - org.apache.hbase - hbase-server - ${hbase096.version} - true - - - avro - org.apache.avro - - - jruby-complete - org.jruby - - - asm - asm - - - - - jruby-complete - org.jruby - ${jruby.version} - test - - - - - src/main/java - ../common/test/java - - - ${project.basedir}/../common/test/resources - true - - - - - maven-resources-plugin - - - filter-hbase-conf - - - - - maven-dependency-plugin - - - generate-cached-classpath - - - unpack-common-test-classes - - - - - maven-assembly-plugin - - - copy-hbase-bin - - - - - maven-clean-plugin - - - clean-hbase-files - - - - - maven-deploy-plugin - - - default-deploy - none - - - - - - - - - hadoop1 - - - - hadoop.profile - 1.0 - - - - - ${hbase096.core.version}-hadoop1 - - - - - - maven-surefire-plugin - - - ${project.basedir}/conf - - - - - - - - - org.apache.hadoop - hadoop-core - - - org.apache.hadoop - hadoop-client - - - - io.netty - netty - - - - - - hadoop2 - - - - !hadoop.profile - - - - - - org.apache.hadoop - hadoop-annotations - true - - - org.apache.hadoop - hadoop-hdfs - true - - - org.apache.hadoop - hadoop-client - true - - - - - diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java deleted file mode 100644 index 6c385d56f9..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_96.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; - -public class HBaseAdmin0_96 implements AdminMask -{ - - private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_96.class); - - private final HBaseAdmin adm; - - public HBaseAdmin0_96(HBaseAdmin adm) - { - this.adm = adm; - } - - @Override - public void clearTable(String tableName, long timestamp) throws IOException - { - if (!adm.tableExists(tableName)) { - log.debug("clearStorage() called before table {} was created, skipping.", tableName); - return; - } - -// long before = System.currentTimeMillis(); -// try { -// adm.disableTable(tableName); -// adm.deleteTable(tableName); -// } catch (IOException e) { -// throw new PermanentBackendException(e); -// } -// ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); -// long after = System.currentTimeMillis(); -// logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before); - - - // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than - // disabling and deleting tables. - HTable table = null; - - try { - table = new HTable(adm.getConfiguration(), tableName); - - Scan scan = new Scan(); - scan.setBatch(100); - scan.setCacheBlocks(false); - scan.setCaching(2000); - scan.setTimeRange(0, Long.MAX_VALUE); - scan.setMaxVersions(1); - - ResultScanner scanner = null; - - try { - scanner = table.getScanner(scan); - - for (Result res : scanner) { - Delete d = new Delete(res.getRow()); - - d.setTimestamp(timestamp); - table.delete(d); - } - } finally { - IOUtils.closeQuietly(scanner); - } - } finally { - IOUtils.closeQuietly(table); - } - } - - @Override - public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException - { - return adm.getTableDescriptor(tableName.getBytes()); - } - - @Override - public boolean tableExists(String tableName) throws IOException - { - return adm.tableExists(tableName); - } - - @Override - public void createTable(HTableDescriptor desc) throws IOException - { - adm.createTable(desc); - } - - @Override - public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException - { - adm.createTable(desc, startKey, endKey, numRegions); - } - - @Override - public int getEstimatedRegionServerCount() - { - int serverCount = -1; - try { - serverCount = adm.getClusterStatus().getServers().size(); - log.debug("Read {} servers from HBase ClusterStatus", serverCount); - } catch (IOException e) { - log.debug("Unable to retrieve HBase cluster status", e); - } - return serverCount; - } - - @Override - public void disableTable(String tableName) throws IOException - { - adm.disableTable(tableName); - } - - @Override - public void enableTable(String tableName) throws IOException - { - adm.enableTable(tableName); - } - - @Override - public boolean isTableDisabled(String tableName) throws IOException - { - return adm.isTableDisabled(tableName); - } - - @Override - public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException - { - adm.addColumn(tableName, columnDescriptor); - } - - @Override - public void close() throws IOException - { - adm.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java deleted file mode 100644 index 14e6376140..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_96.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.io.compress.Compression; - -public class HBaseCompat0_96 implements HBaseCompat { - - @Override - public void setCompression(HColumnDescriptor cd, String algo) { - cd.setCompressionType(Compression.Algorithm.valueOf(algo)); - } - - @Override - public HTableDescriptor newTableDescriptor(String tableName) { - TableName tn = TableName.valueOf(tableName); - return new HTableDescriptor(tn); - } - - @Override - public ConnectionMask createConnection(Configuration conf) throws IOException - { - return new HConnection0_96(HConnectionManager.createConnection(conf)); - } - - @Override - public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc) - { - tdesc.addFamily(cdesc); - } - - @Override - public void setTimestamp(Delete d, long timestamp) - { - d.setTimestamp(timestamp); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java deleted file mode 100644 index 94b0e20a05..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_96.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; - -public class HConnection0_96 implements ConnectionMask -{ - - private final HConnection cnx; - - public HConnection0_96(HConnection cnx) - { - this.cnx = cnx; - } - - @Override - public TableMask getTable(String name) throws IOException - { - return new HTable0_96(cnx.getTable(name)); - } - - @Override - public AdminMask getAdmin() throws IOException - { - return new HBaseAdmin0_96(new HBaseAdmin(cnx)); - } - - @Override - public void close() throws IOException - { - cnx.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java b/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java deleted file mode 100644 index 4845c5c9eb..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_96.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Row; -import org.apache.hadoop.hbase.client.Scan; - -public class HTable0_96 implements TableMask -{ - private final HTableInterface table; - - public HTable0_96(HTableInterface table) - { - this.table = table; - } - - @Override - public ResultScanner getScanner(Scan filter) throws IOException - { - return table.getScanner(filter); - } - - @Override - public Result[] get(List gets) throws IOException - { - return table.get(gets); - } - - @Override - public void batch(List writes, Object[] results) throws IOException, InterruptedException - { - table.batch(writes, results); - table.flushCommits(); - } - - @Override - public void close() throws IOException - { - table.close(); - } -} diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb deleted file mode 100755 index f7e1ff6793..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/get-active-master.rb +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env hbase-jruby -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Prints the hostname of the machine running the active master. - -include Java -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.ServerName -import org.apache.hadoop.hbase.zookeeper.ZKUtil -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher - -# disable debug/info logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) -org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) - -config = HBaseConfiguration.create - -zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) -begin - master_address = ZKUtil.getData(zk, zk.masterAddressZNode) - if master_address - puts ServerName.parseFrom(master_address).getHostname() - else - puts 'Master not running' - end -ensure - zk.close() -end - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh deleted file mode 100755 index 37b5685085..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/graceful_stop.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2011 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Move regions off a server then stop it. Optionally restart and reload. -# Turn off the balancer before running this script. -function usage { - echo "Usage: graceful_stop.sh [--config ] [-d] [-e] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " d|debug Print helpful debug information" - echo " hostname Hostname of server we are to stop" - echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" - exit 1 -} - -if [ $# -lt 1 ]; then - usage -fi - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh -# Get arguments -restart= -reload= -debug= -thrift= -rest= -failfast= -while [ $# -gt 0 ] -do - case "$1" in - --thrift) thrift=true; shift;; - --rest) rest=true; shift;; - --restart) restart=true; shift;; - --reload) reload=true; shift;; - --failfast | -e) failfast=true; shift;; - --debug | -d) debug="--debug"; shift;; - --) shift; break;; - -*) usage ;; - *) break;; # terminate while loop - esac -done - -# "$@" contains the rest. Must be at least the hostname left. -if [ $# -lt 1 ]; then - usage -fi - -# Emit a log line w/ iso8901 date prefixed -log() { - echo `date +%Y-%m-%dT%H:%M:%S` $1 -} - -# See if we should set fail fast before we do anything. -if [ "$failfast" != "" ]; then - log "Set failfast, will exit immediately if any command exits with non-zero status" - set -e -fi - -hostname=$1 -filename="/tmp/$hostname" - -log "Disabling load balancer" -HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` -log "Previous balancer state was $HBASE_BALANCER_STATE" - -log "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname -log "Unloaded $hostname region(s)" - -# Stop the server(s). Have to put hostname into its own little file for hbase-daemons.sh -hosts="/tmp/$(basename $0).$$.tmp" -echo $hostname >> $hosts -if [ "$thrift" != "" ]; then - log "Stopping thrift" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift -fi -if [ "$rest" != "" ]; then - log "Stopping rest" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest -fi -log "Stopping regionserver" -"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver - -if [ "$restart" != "" ]; then - log "Restarting regionserver" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver - if [ "$thrift" != "" ]; then - log "Restarting thrift" - # -b 0.0.0.0 says listen on all interfaces rather than just default. - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 - fi - if [ "$rest" != "" ]; then - log "Restarting rest" - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest - fi - if [ "$reload" != "" ]; then - log "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname - log "Reloaded $hostname region(s)" - fi -fi - -# Restore balancer state -if [ $HBASE_BALANCER_STATE != "false" ]; then - log "Restoring balancer state to " $HBASE_BALANCER_STATE - log "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null -fi - -# Cleanup tmp files. -trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase deleted file mode 100755 index 6d59143674..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase +++ /dev/null @@ -1,381 +0,0 @@ -#! /usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# The hbase command script. Based on the hadoop command script putting -# in hbase classes, libs and configurations ahead of hadoop's. -# -# TODO: Narrow the amount of duplicated code. -# -# Environment Variables: -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HBASE_CLASSPATH Extra Java CLASSPATH entries. -# -# HBASE_CLASSPATH_PREFIX Extra Java CLASSPATH entries that should be -# prefixed to the system classpath. -# -# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding -# native libraries. -# -# HBASE_OPTS Extra Java runtime options. -# -# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -# -# HBASE_ROOT_LOGGER The root appender. Default is INFO,console -# -# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. -# Defaults to the jar packaged with HBase. -# -# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -# Empty by default. -# -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -MVN="mvn" -if [ "$M2_HOME" != "" ]; then - MVN="${M2_HOME}"/bin/mvn -fi - -# Detect if we are in hbase sources dir -in_dev_env=false -if [ -d "${HBASE_HOME}/target" ]; then - in_dev_env=true -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hbase [] []" - echo "Options:" - echo " --config DIR Configuration direction to use. Default: ./conf" - echo " --hosts HOSTS Override the list in 'regionservers' file" - echo "" - echo "Commands:" - echo "Some commands take arguments. Pass no args or -h for usage." - echo " shell Run the HBase shell" - echo " hbck Run the hbase 'fsck' tool" - echo " hlog Write-ahead-log analyzer" - echo " hfile Store file analyzer" - echo " zkcli Run the ZooKeeper shell" - echo " upgrade Upgrade hbase" - echo " master Run an HBase HMaster node" - echo " regionserver Run an HBase HRegionServer node" - echo " zookeeper Run a Zookeeper server" - echo " rest Run an HBase REST server" - echo " thrift Run the HBase Thrift server" - echo " thrift2 Run the HBase Thrift2 server" - echo " clean Run the HBase clean up script" - echo " classpath Dump hbase CLASSPATH" - echo " mapredcp Dump CLASSPATH entries required by mapreduce" - echo " version Print the version" - echo " CLASSNAME Run the class named CLASSNAME" - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -# override default settings for this command, if applicable -if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then - . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" -fi - -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX -fi - -# so that filenames w/ spaces are handled correctly in loops below -ORIG_IFS=$IFS -IFS= - -# CLASSPATH initially contains $HBASE_CONF_DIR -CLASSPATH="${HBASE_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -add_to_cp_if_exists() { - if [ -d "$@" ]; then - CLASSPATH=${CLASSPATH}:"$@" - fi -} - -# For releases, add hbase & webapps to CLASSPATH -# Webapps must come first else it messes up Jetty -if [ -d "$HBASE_HOME/hbase-webapps" ]; then - add_to_cp_if_exists "${HBASE_HOME}" -fi -#add if we are in a dev environment -if [ -d "$HBASE_HOME/hbase-server/target/hbase-webapps" ]; then - add_to_cp_if_exists "${HBASE_HOME}/hbase-server/target" -fi - -add_maven_deps_to_classpath() { - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ]; then - # ------------------- - # Titan build support - # ------------------- - # Check for Maven command line options file. - # If titan-hbase has already been built, then this file will contain options - # to target the version of Hadoop used during compilation. This is - # necessary when the user compiles titan-hbase with a non-default Hadoop - # version (e.g. 2.2.0). If we omitted this check, then we'd try to start - # HBase with the default Hadoop version (currentl 1.0.4), and it would break. - MVN_OPTS= - optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" - unset IFS - if [ -f "${optf}" ]; then - MVN_OPTS=`cat "${optf}"` - fi - "$MVN" -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - - -#Add the development env class path stuff -if $in_dev_env; then - add_maven_deps_to_classpath -fi - -#add the hbase jars for each module -for f in $HBASE_HOME/hbase-jars/hbase*.jar; do - if [[ $f = *sources.jar ]] - then - : # Skip sources.jar - elif [ -f $f ] - then - CLASSPATH=${CLASSPATH}:$f; - fi -done - -# Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# default log directory & file -if [ "$HBASE_LOG_DIR" = "" ]; then - HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -if [ "$HBASE_LOGFILE" = "" ]; then - HBASE_LOGFILE='hbase.log' -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - -function append_path() { - if [ -z "$1" ]; then - echo $2 - else - echo $1:$2 - fi -} - -JAVA_PLATFORM="" - -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) -if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) -fi - -if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then - if [ -z $JAVA_PLATFORM ]; then - JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - fi - if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) - fi - - if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) - fi -fi - -# Add user-specified CLASSPATH last -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} -fi - -# Add user-specified CLASSPATH prefix first -if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then - CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH} -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -# restore ordinary behaviour -unset IFS - -#Set the right GC options based on the what we are running -declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") -for cmd in ${server_cmds[@]}; do - if [[ $cmd == $COMMAND ]]; then - server=true - break - fi -done - -if [[ $server ]]; then - HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" -else - HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" -fi - -# figure out which class to run -if [ "$COMMAND" = "shell" ] ; then - # eg export JRUBY_HOME=/usr/local/share/jruby - if [ "$JRUBY_HOME" != "" ] ; then - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi - # find the hbase ruby sources - HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" -elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -elif [ "$COMMAND" = "hlog" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' -elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFile' -elif [ "$COMMAND" = "zkcli" ] ; then - CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer" -elif [ "$COMMAND" = "upgrade" ] ; then - CLASS="org.apache.hadoop.hbase.migration.UpgradeTo96" -elif [ "$COMMAND" = "master" ] ; then - CLASS='org.apache.hadoop.hbase.master.HMaster' - if [ "$1" != "stop" ] && [ "$1" != "clear" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" - fi -elif [ "$COMMAND" = "regionserver" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" - fi -elif [ "$COMMAND" = "thrift" ] ; then - CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "thrift2" ] ; then - CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "rest" ] ; then - CLASS='org.apache.hadoop.hbase.rest.RESTServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" - fi -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" - fi -elif [ "$COMMAND" = "clean" ] ; then - case $1 in - --cleanZk|--cleanHdfs|--cleanAll) - matches="yes" ;; - *) ;; - esac - if [ $# -ne 1 -o "$matches" = "" ]; then - echo "Usage: hbase clean (--cleanZk|--cleanHdfs|--cleanAll)" - echo "Options: " - echo " --cleanZk cleans hbase related data from zookeeper." - echo " --cleanHdfs cleans hbase related data from hdfs." - echo " --cleanAll cleans hbase related data from both zookeeper and hdfs." - exit 1; - fi - "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ - exit $? -elif [ "$COMMAND" = "mapredcp" ] ; then - CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' -elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit 0 -elif [ "$COMMAND" = "version" ] ; then - CLASS='org.apache.hadoop.hbase.util.VersionInfo' -else - CLASS=$COMMAND -fi - -# Have JVM dump heap if we run out of memory. Files will be 'launch directory' -# and are named like the following: java_pid21612.hprof. Apparently it doesn't -# 'cost' to have this flag enabled. Its a 1.6 flag only. See: -# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" -HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" -HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" -HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" -fi - -# Enable security logging on the master and regionserver only -if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,RFAS}" -else - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" -fi - -# Exec unless HBASE_NOEXEC is set. -export CLASSPATH -if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" -else - exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" -fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh deleted file mode 100755 index 8f1e4b479f..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-cleanup.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# cleans hbase related data from zookeeper and hdfs if no hbase process is alive. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# - -usage="Usage: hbase-cleanup.sh (zk|hdfs|all)" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -case $1 in - --cleanZk|--cleanHdfs|--cleanAll) - matches="yes" ;; - *) ;; -esac -if [ $# -ne 1 -o "$matches" = "" ]; then - echo $usage - exit 1; -fi - -format_option=$1; - -zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` -if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - -hrootdir=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.rootdir` -if [ "$hrootdir" == "null" ]; then hrootdir="file:///tmp/hbase-${USER}/hbase"; fi - -check_for_znodes() { - command=$1; - case $command in - regionservers) - zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zchild" == "null" ]; then zchild="rs"; fi - ;; - backupmasters) - zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.backup.masters` - if [ "$zchild" == "null" ]; then zchild="backup-masters"; fi - ;; - esac - znodes=`"$bin"/hbase zkcli ls $zparent/$zchild 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - if [ "$znodes" != "" ]; then - echo -n "ZNode(s) [${znodes}] of $command are not expired. Exiting without cleaning hbase data." - echo #force a newline - exit 1; - else - echo -n "All ZNode(s) of $command are expired." - fi - echo #force a newline -} - -execute_zk_command() { - command=$1; - "$bin"/hbase zkcli $command 2>&1 -} - -execute_hdfs_command() { - command=$1; - "$bin"/hbase org.apache.hadoop.fs.FsShell $command 2>&1 -} - -clean_up() { - case $1 in - --cleanZk) - execute_zk_command "rmr ${zparent}"; - ;; - --cleanHdfs) - execute_hdfs_command "-rmr ${hrootdir}" - ;; - --cleanAll) - execute_zk_command "rmr ${zparent}"; - execute_hdfs_command "-rmr ${hrootdir}" - ;; - *) - ;; - esac -} - -check_znode_exists() { - command=$1 - "$bin"/hbase zkcli stat $command 2>&1 | grep "Node does not exist\|Connection refused" -} - -check_znode_exists $zparent -if [ $? -ne 0 ]; then - # make sure the online region server(s) znode(s) have been deleted before continuing - check_for_znodes regionservers - # make sure the backup master(s) znode(s) has been deleted before continuing - check_for_znodes backupmasters - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - check_znode_exists $zmaster - if [ $? -ne 0 ]; then - echo -n "Master ZNode is not expired. Exiting without cleaning hbase data." - echo #force a new line - exit 1 - else - echo "Active Master ZNode also expired." - fi - echo #force a newline -else - echo "HBase parent znode ${zparent} does not exist." -fi - -# cleans zookeeper and/or hdfs data. -clean_up $format_option diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh deleted file mode 100755 index 0a474f7304..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-common.sh +++ /dev/null @@ -1,41 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## - -#Shared function to wait for a process end. Take the pid and the command name as parameters -waitForProcessEnd() { - pidKilled=$1 - commandName=$2 - processedAt=`date +%s` - while kill -0 $pidKilled > /dev/null 2>&1; - do - echo -n "." - sleep 1; - # if process persists more than $HBASE_STOP_TIMEOUT (default 1200 sec) no mercy - if [ $(( `date +%s` - $processedAt )) -gt ${HBASE_STOP_TIMEOUT:-1200} ]; then - break; - fi - done - # process still there : kill -9 - if kill -0 $pidKilled > /dev/null 2>&1; then - echo -n force stopping $commandName with kill -9 $pidKilled - $JAVA_HOME/bin/jstack -l $pidKilled > "$logout" 2>&1 - kill -9 $pidKilled > /dev/null 2>&1 - fi - # Add a CR after we're done w/ dots. - echo -} diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh deleted file mode 100755 index 0300fc5ca3..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-config.sh +++ /dev/null @@ -1,155 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# included in all the hbase scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* -# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -# resolve links - "${BASH_SOURCE-$0}" may be a softlink - -this="${BASH_SOURCE-$0}" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin">/dev/null; pwd` -this="$bin/$script" - -# the root of the hbase installation -if [ -z "$HBASE_HOME" ]; then - export HBASE_HOME=`dirname "$this"`/.. -fi - -#check to see if the conf dir or hbase home are given as an optional arguments -while [ $# -gt 1 ] -do - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HBASE_CONF_DIR=$confdir - elif [ "--hosts" = "$1" ] - then - shift - hosts=$1 - shift - HBASE_REGIONSERVERS=$hosts - else - # Presume we are at end of options and break - break - fi -done - -# Allow alternate hbase conf dir location. -HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" -# List of hbase regions servers. -HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" -# List of hbase secondary masters. -HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" -# Thrift JMX opts -if [[ -n "$HBASE_JMX_OPTS" && -z "$HBASE_THRIFT_JMX_OPTS" ]]; then - HBASE_THRIFT_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10103" -fi -# Thrift opts -if [ -z "$HBASE_THRIFT_OPTS" ]; then - export HBASE_THRIFT_OPTS="$HBASE_THRIFT_JMX_OPTS" -fi - -# REST JMX opts -if [[ -n "$HBASE_JMX_OPTS" && -z "$HBASE_REST_JMX_OPTS" ]]; then - HBASE_REST_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10105" -fi -# REST opts -if [ -z "$HBASE_REST_OPTS" ]; then - export HBASE_REST_OPTS="$HBASE_REST_JMX_OPTS" -fi - -# Source the hbase-env.sh. Will have JAVA_HOME defined. -# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. -if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then - . "${HBASE_CONF_DIR}/hbase-env.sh" - export HBASE_ENV_INIT="true" -fi - -# Set default value for regionserver uid if not present -if [ -z "$HBASE_REGIONSERVER_UID" ]; then - HBASE_REGIONSERVER_UID="hbase" -fi - -# Verify if hbase has the mlock agent -if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then - MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so" - if [ ! -f "$MLOCK_AGENT" ]; then - cat 1>&2 <&2 < http://java.sun.com/javase/downloads/ < | -| | -| HBase requires Java 1.6 or later. | -| NOTE: This script will find Sun Java whether you install using the | -| binary or the RPM based installer. | -+======================================================================+ -EOF - exit 1 - fi -fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh deleted file mode 100755 index caa74d8d7b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemon.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Runs a Hadoop hbase command as a daemon. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_LOG_DIR Where log files are stored. PWD by default. -# HBASE_PID_DIR The pid files are stored. /tmp by default. -# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default -# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. -# HBASE_STOP_TIMEOUT Time, in seconds, after which we kill -9 the server if it has not stopped. -# Default 1200 seconds. -# -# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh - -usage="Usage: hbase-daemon.sh [--config ]\ - (start|stop|restart|autorestart) \ - " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh -. "$bin"/hbase-common.sh - -# get arguments -startStop=$1 -shift - -command=$1 -shift - -hbase_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" - num=$prev - done - mv -f "$log" "$log.$num"; - fi -} - -cleanZNode() { - if [ -f $HBASE_ZNODE_FILE ]; then - if [ "$command" = "master" ]; then - $bin/hbase master clear > /dev/null 2>&1 - else - #call ZK to delete the node - ZNODE=`cat $HBASE_ZNODE_FILE` - $bin/hbase zkcli delete $ZNODE > /dev/null 2>&1 - fi - rm $HBASE_ZNODE_FILE - fi -} - -check_before_start(){ - #ckeck if the process is not running - mkdir -p "$HBASE_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi -} - -wait_until_done () -{ - p=$1 - cnt=${HBASE_SLAVE_TIMEOUT:-300} - origcnt=$cnt - while kill -0 $p > /dev/null 2>&1; do - if [ $cnt -gt 1 ]; then - cnt=`expr $cnt - 1` - sleep 1 - else - echo "Process did not complete after $origcnt seconds, killing." - kill -9 $p - exit 1 - fi - done - return 0 -} - -# get log directory -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_PID_DIR" = "" ]; then - HBASE_PID_DIR=/tmp -fi - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -# Some variables -# Work out java location so can print version into log. -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"} -export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"} -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid -export HBASE_ZNODE_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.znode -export HBASE_START_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.autorestart - -if [ -n "$SERVER_GC_OPTS" ]; then - export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi -if [ -n "$CLIENT_GC_OPTS" ]; then - export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} -fi - -# Set default scheduling priority -if [ "$HBASE_NICENESS" = "" ]; then - export HBASE_NICENESS=0 -fi - -thiscmd=$0 -args=$@ - -case $startStop in - -(start) - check_before_start - hbase_rotate_log $logout - hbase_rotate_log $loggc - echo starting $command, logging to $logout - nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args < /dev/null > ${logout} 2>&1 & - sleep 1; head "${logout}" - ;; - -(autorestart) - check_before_start - hbase_rotate_log $logout - hbase_rotate_log $loggc - nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_autorestart $command $args < /dev/null > ${logout} 2>&1 & - ;; - -(internal_start) - # Add to the command log file vital stats on our environment. - echo "`date` Starting $command on `hostname`" >> $loglog - echo "`ulimit -a`" >> $loglog 2>&1 - nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - $command "$@" start >> "$logout" 2>&1 & - echo $! > $pid - wait - cleanZNode - ;; - -(internal_autorestart) - touch "$HBASE_START_FILE" - #keep starting the command until asked to stop. Reloop on software crash - while true - do - lastLaunchDate=`date +%s` - $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args - - #if the file does not exist it means that it was not stopped properly by the stop command - if [ ! -f "$HBASE_START_FILE" ]; then - exit 1 - fi - - #if the cluster is being stopped then do not restart it again. - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - zshutdown=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.state` - if [ "$zshutdown" == "null" ]; then zshutdown="shutdown"; fi - zFullShutdown=$zparent/$zshutdown - $bin/hbase zkcli stat $zFullShutdown 2>&1 | grep "Node does not exist" 1>/dev/null 2>&1 - #grep returns 0 if it found something, 1 otherwise - if [ $? -eq 0 ]; then - exit 1 - fi - - #If ZooKeeper cannot be found, then do not restart - $bin/hbase zkcli stat $zFullShutdown 2>&1 | grep Exception | grep ConnectionLoss 1>/dev/null 2>&1 - if [ $? -eq 0 ]; then - exit 1 - fi - - #if it was launched less than 5 minutes ago, then wait for 5 minutes before starting it again. - curDate=`date +%s` - limitDate=`expr $lastLaunchDate + 300` - if [ $limitDate -gt $curDate ]; then - sleep 300 - fi - done - ;; - -(stop) - rm -f "$HBASE_START_FILE" - if [ -f $pid ]; then - pidToKill=`cat $pid` - # kill -0 == see if the PID exists - if kill -0 $pidToKill > /dev/null 2>&1; then - echo -n stopping $command - echo "`date` Terminating $command" >> $loglog - kill $pidToKill > /dev/null 2>&1 - waitForProcessEnd $pidToKill $command - rm $pid - else - retval=$? - echo no $command to stop because kill -0 of pid $pidToKill failed with status $retval - fi - else - echo no $command to stop because no pid file $pid - fi - ;; - -(restart) - # stop the command - $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & - wait_until_done $! - # wait a user-specified sleep period - sp=${HBASE_RESTART_SLEEP:-3} - if [ $sp -gt 0 ]; then - sleep $sp - fi - # start the command - $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & - wait_until_done $! - ;; - -(*) - echo $usage - exit 1 - ;; -esac diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh deleted file mode 100755 index 843eaaa74f..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-daemons.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [--hosts regionserversfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. $bin/hbase-config.sh - -remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" -args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" - -command=$2 -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (master-backup) - exec "$bin/master-backup.sh" $args - ;; - (*) - exec "$bin/regionservers.sh" $args - ;; -esac - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby b/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby deleted file mode 100755 index 37bce46f6b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hbase-jruby +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - - -`dirname $0`/hbase org.jruby.Main $* - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb deleted file mode 100755 index dc1feccfae..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/hirb.rb +++ /dev/null @@ -1,190 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports -# and hbase commands and then loads jirb. Outputs a banner that tells user -# where to find help, shell version, and loads up a custom hirb. - -# TODO: Interrupt a table creation or a connection to a bad master. Currently -# has to time out. Below we've set down the retries for rpc and hbase but -# still can be annoying (And there seem to be times when we'll retry for -# ever regardless) -# TODO: Add support for listing and manipulating catalog tables, etc. -# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes - -# Run the java magic include and import basic HBase types that will help ease -# hbase hacking. -include Java - -# Some goodies for hirb. Should these be left up to the user's discretion? -require 'irb/completion' -require 'pathname' - -# Add the directory names in hbase.jruby.sources commandline option -# to the ruby load path so I can load up my HBase ruby modules -sources = java.lang.System.getProperty('hbase.ruby.sources') -$LOAD_PATH.unshift Pathname.new(sources) - -# -# FIXME: Switch args processing to getopt -# -# See if there are args for this shell. If any, read and then strip from ARGV -# so they don't go through to irb. Output shell 'usage' if user types '--help' -cmdline_help = </dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [--config ] [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1" - exit -fi - -. "$bin"/hbase-config.sh - -# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] -export HBASE_MASTER_OPTS=" " - -run_master () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_MASTER_ARGS="\ - -D hbase.master.port=`expr 60000 + $DN` \ - -D hbase.master.info.port=`expr 60010 + $DN` \ - --backup" - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 master $HBASE_MASTER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_master $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh deleted file mode 100755 index bf4372e21d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/local-regionservers.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple regionservers on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 100 regionservers (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [--config ] [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1 2" - exit -fi - -. "$bin"/hbase-config.sh - -# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] -export HBASE_REGIONSERVER_OPTS=" " - -run_regionserver () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 60200 + $DN` \ - -D hbase.regionserver.info.port=`expr 60300 + $DN`" - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 regionserver $HBASE_REGIONSERVER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_regionserver $cmd $i -done diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh deleted file mode 100755 index 114757c763..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/master-backup.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2010 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all backup master hosts. -# -# Environment Variables -# -# HBASE_BACKUP_MASTERS File naming remote hosts. -# Default is ${HBASE_CONF_DIR}/backup-masters -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the master backup file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_BACKUP_MASTERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_BACKUP_MASTERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" - else - export HOSTLIST="${HBASE_BACKUP_MASTERS}" - fi -fi - - -args=${@// /\\ } -args=${args/master-backup/master} - -if [ -f $HOSTLIST ]; then - for hmaster in `cat "$HOSTLIST"`; do - ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ - 2>&1 | sed "s/^/$hmaster: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb deleted file mode 100755 index cc57456f61..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_mover.rb +++ /dev/null @@ -1,487 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Moves regions. Will confirm region access in current location and will -# not move a new region until successful confirm of region loading in new -# location. Presumes balancer is disabled when we run (not harmful if its -# on but this script and balancer will end up fighting each other). -# Does not work for case of multiple regionservers all running on the -# one node. -require 'optparse' -include Java -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.Get -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.HConnectionManager -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.util.Writables -import org.apache.hadoop.conf.Configuration -import org.apache.commons.logging.Log -import org.apache.commons.logging.LogFactory -import org.apache.hadoop.hbase.protobuf.ProtobufUtil -import org.apache.hadoop.hbase.ServerName -import org.apache.hadoop.hbase.HRegionInfo - -# Name of this script -NAME = "region_mover" - -# Get meta table reference -def getMetaTable(config) - # Keep meta reference in ruby global - if not $META - $META = HTable.new(config, HConstants::META_TABLE_NAME) - end - return $META -end - -# Get table instance. -# Maintains cache of table instances. -def getTable(config, name) - # Keep dictionary of tables in ruby global - if not $TABLES - $TABLES = {} - end - key = name.toString() - if not $TABLES[key] - $TABLES[key] = HTable.new(config, name) - end - return $TABLES[key] -end - -def closeTables() - if not $TABLES - return - end - - $LOG.info("Close all tables") - $TABLES.each do |name, table| - $TABLES.delete(name) - table.close() - end -end - -# Returns true if passed region is still on 'original' when we look at .META. -def isSameServer(admin, r, original) - server = getServerNameForRegion(admin, r) - return false unless server - return true unless original - return server == original -end - -class RubyAbortable - include org.apache.hadoop.hbase.Abortable - def abort(why, e) - puts "ABORTED! why=" + why + ", e=" + e.to_s - end -end - -# Get servername that is up in .META.; this is hostname + port + startcode comma-delimited. -# Can return nil -def getServerNameForRegion(admin, r) - if r.isMetaRegion() - # Hack - zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(admin.getConfiguration(), "region_mover", nil) - begin - tracker = org.apache.hadoop.hbase.zookeeper.MetaRegionTracker.new(zkw, RubyAbortable.new()) - tracker.start() - while not tracker.isLocationAvailable() - sleep 0.1 - end - # Make a fake servername by appending ',' - metaServer = tracker.getMetaRegionLocation().toString() + "," - tracker.stop() - return metaServer - ensure - zkw.close() - end - end - table = nil - table = getMetaTable(admin.getConfiguration()) - g = Get.new(r.getRegionName()) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - result = table.get(g) - server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) - startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) - return nil unless server - return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s -end - -# Trys to scan a row from passed region -# Throws exception if can't -def isSuccessfulScan(admin, r) - scan = Scan.new(r.getStartKey()) - scan.setBatch(1) - scan.setCaching(1) - scan.setFilter(FirstKeyOnlyFilter.new()) - table = getTable(admin.getConfiguration(), r.getTableName()) - scanner = table.getScanner(scan) - begin - results = scanner.next() - # We might scan into next region, this might be an empty table. - # But if no exception, presume scanning is working. - ensure - scanner.close() - # Do not close the htable. It is cached in $TABLES and - # may be reused in moving another region of same table. - # table.close() - end -end - -# Check region has moved successful and is indeed hosted on another server -# Wait until that is the case. -def move(admin, r, newServer, original) - # Now move it. Do it in a loop so can retry if fail. Have seen issue where - # we tried move region but failed and retry put it back on old location; - # retry in this case. - retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) - count = 0 - same = true - start = Time.now - while count < retries and same - if count > 0 - $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s) - end - count = count + 1 - begin - admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) - rescue java.lang.reflect.UndeclaredThrowableException => e - $LOG.info("Exception moving " + r.getEncodedName() + - "; split/moved? Continuing: " + e) - return - end - # Wait till its up on new server before moving on - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.move.wait.max", 60) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - same = isSameServer(admin, r, original) - break unless same - sleep 0.1 - end - end - raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same - # Assert can Scan from new location. - isSuccessfulScan(admin, r) - $LOG.info("Moved region " + r.getRegionNameAsString() + " cost: " + - java.lang.String.format("%.3f", (Time.now - start))) -end - -# Return the hostname portion of a servername (all up to first ',') -def getHostnamePortFromServerName(serverName) - parts = serverName.split(',') - return parts[0] + ":" + parts[1] -end - -# Return the hostname:port out of a servername (all up to first ',') -def getHostnameFromServerName(serverName) - return serverName.split(',')[0] -end - -# Return array of servernames where servername is hostname+port+startcode -# comma-delimited -def getServers(admin) - serverInfos = admin.getClusterStatus().getServerInfo() - servers = [] - for server in serverInfos - servers << server.getServerName() - end - return servers -end - -# Remove the servername whose hostname portion matches from the passed -# array of servers. Returns as side-effect the servername removed. -def stripServer(servers, hostname) - count = servers.length - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = servers.delete(server) - end - end - # Check server to exclude is actually present - raise RuntimeError, "Server %s not online" % hostname unless servers.length < count - return servername -end - -# Returns a new serverlist that excludes the servername whose hostname portion -# matches from the passed array of servers. -def stripExcludes(servers, excludefile) - excludes = readExcludes(excludefile) - servers = servers.find_all{|server| !excludes.contains(getHostnameFromServerName(server)) } - # return updated servers list - return servers -end - - -# Return servername that matches passed hostname -def getServerName(servers, hostname) - servername = nil - for server in servers - if getHostnameFromServerName(server) == hostname - servername = server - break - end - end - raise ArgumentError, "Server %s not online" % hostname unless servername - return servername -end - -# Create a logger and disable the DEBUG-level annoying client logging -def configureLogging(options) - apacheLogger = LogFactory.getLog(NAME) - # Configure log4j to not spew so much - unless (options[:debug]) - logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase.client") - logger.setLevel(org.apache.log4j.Level::INFO) - end - return apacheLogger -end - -# Get configuration instance -def getConfiguration() - config = HBaseConfiguration.create() - # No prefetching on .META. - config.setInt("hbase.client.prefetch.limit", 1) - # Make a config that retries at short intervals many times - config.setInt("hbase.client.pause", 500) - config.setInt("hbase.client.retries.number", 100) - return config -end - -# Now get list of regions on targetServer -def getRegions(config, servername) - connection = HConnectionManager::getConnection(config); - return ProtobufUtil::getOnlineRegions(connection.getAdmin(ServerName.new(servername))); -end - -def deleteFile(filename) - f = java.io.File.new(filename) - f.delete() if f.exists() -end - -# Write HRegionInfo to file -# Need to serialize in case non-printable characters. -# Format is count of regionnames followed by serialized regionnames. -def writeFile(filename, regions) - fos = java.io.FileOutputStream.new(filename) - dos = java.io.DataOutputStream.new(fos) - # Write out a count of region names - dos.writeInt(regions.size()) - # Write actual region names. - for r in regions - Bytes.writeByteArray(dos, r.toByteArray()) - end - dos.close() -end - -# See writeFile above. -# Returns array of HRegionInfos -def readFile(filename) - f = java.io.File.new(filename) - return java.util.ArrayList.new() unless f.exists() - fis = java.io.FileInputStream.new(f) - dis = java.io.DataInputStream.new(fis) - # Read count of regions - count = dis.readInt() - regions = java.util.ArrayList.new(count) - index = 0 - while index < count - regions.add(HRegionInfo.parseFromOrNull(Bytes.readByteArray(dis))) - index = index + 1 - end - dis.close() - return regions -end - -# Move regions off the passed hostname -def unloadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Clean up any old files. - filename = getFilename(options, hostname) - deleteFile(filename) - # Get an admin instance - admin = HBaseAdmin.new(config) - servers = getServers(admin) - # Remove the server we are unloading from from list of servers. - # Side-effect is the servername that matches this hostname - servername = stripServer(servers, hostname) - - # Remove the servers in our exclude list from list of servers. - servers = stripExcludes(servers, options[:excludesFile]) - puts "Valid region move targets: ", servers - movedRegions = java.util.ArrayList.new() - while true - rs = getRegions(config, servername) - break if rs.length == 0 - count = 0 - $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + - " during this cycle"); - for r in rs - # Get a random server to move the region to. - server = servers[rand(servers.length)] - $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + - count.to_s + " of " + rs.length.to_s + ") from server=" + - servername + " to server=" + server); - count = count + 1 - # Assert we can scan region in its current location - isSuccessfulScan(admin, r) - # Now move it. - move(admin, r, server, servername) - movedRegions.add(r) - end - end - if movedRegions.size() > 0 - # Write out file of regions moved - writeFile(filename, movedRegions) - $LOG.info("Wrote list of moved regions to " + filename) - end -end - -# Move regions to the passed hostname -def loadRegions(options, hostname) - # Get configuration - config = getConfiguration() - # Get an admin instance - admin = HBaseAdmin.new(config) - filename = getFilename(options, hostname) - regions = readFile(filename) - return if regions.isEmpty() - servername = nil - # Wait till server is up - maxWaitInSeconds = admin.getConfiguration.getInt("hbase.serverstart.wait.max", 180) - maxWait = Time.now + maxWaitInSeconds - while Time.now < maxWait - servers = getServers(admin) - begin - servername = getServerName(servers, hostname) - rescue ArgumentError => e - $LOG.info("hostname=" + hostname.to_s + " is not up yet, waiting"); - end - break if servername - sleep 0.5 - end - $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) - count = 0 - # sleep 20s to make sure the rs finished initialization. - sleep 20 - for r in regions - exists = false - begin - isSuccessfulScan(admin, r) - exists = true - rescue org.apache.hadoop.hbase.NotServingRegionException => e - $LOG.info("Failed scan of " + e.message) - end - count = count + 1 - next unless exists - currentServer = getServerNameForRegion(admin, r) - if currentServer and currentServer == servername - $LOG.info("Region " + r.getRegionNameAsString() + " (" + count.to_s + - " of " + regions.length.to_s + ") already on target server=" + servername) - next - end - $LOG.info("Moving region " + r.getRegionNameAsString() + " (" + - count.to_s + " of " + regions.length.to_s + ") from server=" + - currentServer + " to server=" + servername); - move(admin, r, servername, currentServer) - end -end - -# Returns an array of hosts to exclude as region move targets -def readExcludes(filename) - if filename == nil - return java.util.ArrayList.new() - end - if ! File.exist?(filename) - puts "Error: Unable to read host exclude file: ", filename - raise RuntimeError - end - - f = File.new(filename, "r") - # Read excluded hosts list - excludes = java.util.ArrayList.new() - while (line = f.gets) - line.strip! # do an inplace drop of pre and post whitespaces - excludes.add(line) unless line.empty? # exclude empty lines - end - puts "Excluding hosts as region move targets: ", excludes - f.close - - return excludes -end - -def getFilename(options, targetServer) - filename = options[:file] - if not filename - filename = "/tmp/" + ENV['USER'] + targetServer - end - return filename -end - - -# Do command-line parsing -options = {} -optparse = OptionParser.new do |opts| - opts.banner = "Usage: #{NAME}.rb [options] load|unload " - opts.separator 'Load or unload regions by moving one at a time' - options[:file] = nil - opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| - options[:file] = file - end - opts.on('-h', '--help', 'Display usage information') do - puts opts - exit - end - options[:debug] = false - opts.on('-d', '--debug', 'Display extra debug logging') do - options[:debug] = true - end - opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| - options[:excludesFile] = file - end -end -optparse.parse! - -# Check ARGVs -if ARGV.length < 2 - puts optparse - exit 1 -end -hostname = ARGV[1] -if not hostname - opts optparse - exit 2 -end -# Create a logger and save it to ruby global -$LOG = configureLogging(options) -case ARGV[0] - when 'load' - loadRegions(options, hostname) - when 'unload' - unloadRegions(options, hostname) - else - puts optparse - exit 3 -end - -closeTables() diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb deleted file mode 100755 index df35ec6237..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/region_status.rb +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# View the current status of all regions on an HBase cluster. This is -# predominantly used to determined if all the regions in META have been -# onlined yet on startup. -# -# To use this script, run: -# -# ${HBASE_HOME}/bin/hbase org.jruby.Main region_status.rb [wait] [--table ] - - -require 'optparse' - -usage = 'Usage : ./hbase org.jruby.Main region_status.rb [wait]' + - '[--table ]\n' -OptionParser.new do |o| - o.banner = usage - o.on('-t', '--table TABLENAME', 'Only process TABLENAME') do |tablename| - $tablename = tablename - end - o.on('-h', '--help', 'Display help message') { puts o; exit } - o.parse! -end - -SHOULD_WAIT = ARGV[0] == 'wait' -if ARGV[0] and not SHOULD_WAIT - print usage - exit 1 -end - - -require 'java' - -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.MasterNotRunningException -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.client.HTable -import org.apache.hadoop.hbase.client.Scan -import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter -import org.apache.hadoop.hbase.util.Bytes -import org.apache.hadoop.hbase.HRegionInfo -import org.apache.hadoop.hbase.client.MetaScanner -import org.apache.hadoop.hbase.HTableDescriptor -import org.apache.hadoop.hbase.client.HConnectionManager - -# disable debug logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) -org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) - -config = HBaseConfiguration.create -config.set 'fs.default.name', config.get(HConstants::HBASE_DIR) - -# wait until the master is running -admin = nil -while true - begin - admin = HBaseAdmin.new config - break - rescue MasterNotRunningException => e - print 'Waiting for master to start...\n' - sleep 1 - end -end - -meta_count = 0 -server_count = 0 - -# scan META to see how many regions we should have -if $tablename.nil? - scan = Scan.new -else - tableNameMetaPrefix = $tablename + HConstants::META_ROW_DELIMITER.chr - scan = Scan.new( - (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes - ) -end -scan.cache_blocks = false -scan.caching = 10 -scan.setFilter(FirstKeyOnlyFilter.new) -INFO = 'info'.to_java_bytes -REGION_INFO = 'regioninfo'.to_java_bytes -scan.addColumn INFO, REGION_INFO -table = nil -iter = nil -while true - begin - table = HTable.new config, 'hbase:meta'.to_java_bytes - scanner = table.getScanner(scan) - iter = scanner.iterator - break - rescue IOException => ioe - print "Exception trying to scan META: #{ioe}" - sleep 1 - end -end -while iter.hasNext - result = iter.next - rowid = Bytes.toString(result.getRow()) - rowidStr = java.lang.String.new(rowid) - if not $tablename.nil? and not rowidStr.startsWith(tableNameMetaPrefix) - # Gone too far, break - break - end - region = HRegionInfo.getHRegionInfo result - if not region.isOffline - # only include regions that should be online - meta_count += 1 - end -end -scanner.close -# If we're trying to see the status of all HBase tables, we need to include the -# .META. table, that is not included in our scan -if $tablename.nil? - meta_count += 1 -end - -# query the master to see how many regions are on region servers -if not $tablename.nil? - $TableName = HTableDescriptor.new($tablename.to_java_bytes).getTableName() -end -while true - if $tablename.nil? - server_count = admin.getClusterStatus().getRegionsCount() - else - connection = HConnectionManager::getConnection(config); - server_count = MetaScanner::allTableRegions(config, connection, $TableName ,false).size() - end - print "Region Status: #{server_count} / #{meta_count}\n" - if SHOULD_WAIT and server_count < meta_count - #continue this loop until server & meta count match - sleep 10 - else - break - end -end - -exit server_count == meta_count ? 0 : 1 diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh deleted file mode 100755 index 9759f2b00c..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/regionservers.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: regionservers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the regionservers file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_REGIONSERVERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_REGIONSERVERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/regionservers" - else - export HOSTLIST="${HBASE_REGIONSERVERS}" - fi -fi - -for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi -done - -wait diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb deleted file mode 100755 index bc70031cb9..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/replication/copy_tables_desc.rb +++ /dev/null @@ -1,73 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Script to recreate all tables from one cluster to another -# To see usage for this script, run: -# -# ${HBASE_HOME}/bin/hbase org.jruby.Main copy_tables_desc.rb -# - -include Java -import org.apache.commons.logging.LogFactory -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.HConstants -import org.apache.hadoop.hbase.EmptyWatcher -import org.apache.hadoop.hbase.client.HBaseAdmin -import org.apache.hadoop.hbase.HTableDescriptor -import org.apache.hadoop.conf.Configuration - -# Name of this script -NAME = "copy_tables_desc" - -# Print usage for this script -def usage - puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME - exit! -end - -if ARGV.size != 2 - usage -end - -LOG = LogFactory.getLog(NAME) - -parts1 = ARGV[0].split(":") - -parts2 = ARGV[1].split(":") - -c1 = HBaseConfiguration.create() -c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0]) -c1.set("hbase.zookeeper.property.clientPort", parts1[1]) -c1.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts1[2]) - -admin1 = HBaseAdmin.new(c1) - -c2 = HBaseConfiguration.create() -c2.set(HConstants::ZOOKEEPER_QUORUM, parts2[0]) -c2.set("hbase.zookeeper.property.clientPort", parts2[1]) -c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2]) - -admin2 = HBaseAdmin.new(c2) - -for t in admin1.listTables() - admin2.createTable(t) -end - - -puts "All descriptions were copied" diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh deleted file mode 100755 index 97843e8ed4..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/rolling-restart.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -function usage() { - echo $usage - exit 1 -} - - -RR_RS=1 -RR_MASTER=1 -RR_GRACEFUL=0 - -for x in "$@" ; do - case "$x" in - --rs-only|-r) - RR_RS=1 - RR_MASTER=0 - RR_GRACEFUL=0 - ;; - --master-only) - RR_RS=0 - RR_MASTER=1 - RR_GRACEFUL=0 - ;; - --graceful) - RR_RS=0 - RR_MASTER=0 - RR_GRACEFUL=1 - ;; - *) - echo Bad argument: $x - usage - exit 1 - ;; - esac -done - -# quick function to get a value from the HBase config file -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'false' ]; then - if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then - echo Cant do selective rolling restart if not running distributed - exit 1 - fi - "$bin"/hbase-daemon.sh restart master -else - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - - if [ $RR_MASTER -eq 1 ]; then - # stop all masters before re-start to avoid races for master znode - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - echo -n "Waiting for Master ZNode ${zmaster} to expire" - while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do - echo -n "." - sleep 1 - done - echo #force a newline - - # all masters are down, now restart - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup - - echo "Wait a minute for master to come up join cluster" - sleep 60 - - # Master joing cluster will start in cleaning out regions in transition. - # Wait until the master has cleaned out regions in transition before - # giving it a bunch of work to do; master is vulnerable during startup - zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` - if [ "$zunassigned" == "null" ]; then zunassigned="region-in-transition"; fi - zunassigned="$zparent/$zunassigned" - echo -n "Waiting for ${zunassigned} to empty" - while true ; do - unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` - if test 0 -eq ${unassigned} - then - break - else - echo -n " ${unassigned}" - fi - sleep 1 - done - fi - - if [ $RR_RS -eq 1 ]; then - # unlike the masters, roll all regionservers one-at-a-time - export HBASE_SLAVE_PARALLEL=false - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" restart regionserver - fi - - if [ $RR_GRACEFUL -eq 1 ]; then - # gracefully restart all online regionservers - zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` - if [ "$zkrs" == "null" ]; then zkrs="rs"; fi - zkrs="$zparent/$zkrs" - online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - for rs in $online_regionservers - do - rs_parts=(${rs//,/ }) - hostname=${rs_parts[0]} - echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" - sleep 1 - done - fi -fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb deleted file mode 100644 index 87512bf88b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase.rb +++ /dev/null @@ -1,80 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# HBase ruby classes. -# Has wrapper classes for org.apache.hadoop.hbase.client.HBaseAdmin -# and for org.apache.hadoop.hbase.client.HTable. Classes take -# Formatters on construction and outputs any results using -# Formatter methods. These classes are only really for use by -# the hirb.rb HBase Shell script; they don't make much sense elsewhere. -# For example, the exists method on Admin class prints to the formatter -# whether the table exists and returns nil regardless. -include Java - -include_class('java.lang.Integer') {|package,name| "J#{name}" } -include_class('java.lang.Long') {|package,name| "J#{name}" } -include_class('java.lang.Boolean') {|package,name| "J#{name}" } - -module HBaseConstants - COLUMN = "COLUMN" - COLUMNS = "COLUMNS" - TIMESTAMP = "TIMESTAMP" - TIMERANGE = "TIMERANGE" - NAME = org.apache.hadoop.hbase.HConstants::NAME - VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS - IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY - METADATA = org.apache.hadoop.hbase.HConstants::METADATA - STOPROW = "STOPROW" - STARTROW = "STARTROW" - ENDROW = STOPROW - RAW = "RAW" - LIMIT = "LIMIT" - METHOD = "METHOD" - MAXLENGTH = "MAXLENGTH" - CACHE_BLOCKS = "CACHE_BLOCKS" - REPLICATION_SCOPE = "REPLICATION_SCOPE" - INTERVAL = 'INTERVAL' - CACHE = 'CACHE' - FILTER = 'FILTER' - SPLITS = 'SPLITS' - SPLITS_FILE = 'SPLITS_FILE' - SPLITALGO = 'SPLITALGO' - NUMREGIONS = 'NUMREGIONS' - CONFIGURATION = org.apache.hadoop.hbase.HConstants::CONFIGURATION - - # Load constants from hbase java API - def self.promote_constants(constants) - # The constants to import are all in uppercase - constants.each do |c| - next if c =~ /DEFAULT_.*/ || c != c.upcase - next if eval("defined?(#{c})") - eval("#{c} = '#{c}'") - end - end - - promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) - promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) -end - -# Include classes definition -require 'hbase/hbase' -require 'hbase/admin' -require 'hbase/table' -require 'hbase/replication_admin' -require 'hbase/security' diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb deleted file mode 100644 index 8ab2ef4efb..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/admin.rb +++ /dev/null @@ -1,823 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java -java_import java.util.Arrays -java_import org.apache.hadoop.hbase.util.Pair -java_import org.apache.hadoop.hbase.util.RegionSplitter -java_import org.apache.hadoop.hbase.util.Bytes - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class Admin - include HBaseConstants - - def initialize(configuration, formatter) - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - connection = @admin.getConnection() - @conf = configuration - @zk_wrapper = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(configuration, - "admin", nil) - zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() - @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of tables in hbase - def list(regex = ".*") - @admin.getTableNames(regex).to_a - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region flush - def flush(table_or_region_name) - @admin.flush(table_or_region_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family compaction - def compact(table_or_region_name, family = nil) - if family == nil - @admin.compact(table_or_region_name) - else - # We are compacting a column family within a region. - @admin.compact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region or column family major compaction - def major_compact(table_or_region_name, family = nil) - if family == nil - @admin.majorCompact(table_or_region_name) - else - # We are major compacting a column family within a region or table. - @admin.majorCompact(table_or_region_name, family) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a regionserver's HLog roll - def hlog_roll(server_name) - @admin.rollHLogWriter(server_name) - end - - #---------------------------------------------------------------------------------------------- - # Requests a table or region split - def split(table_or_region_name, split_point) - if split_point == nil - @admin.split(table_or_region_name) - else - @admin.split(table_or_region_name, split_point) - end - end - - #---------------------------------------------------------------------------------------------- - # Requests a cluster balance - # Returns true if balancer ran - def balancer() - @admin.balancer() - end - - #---------------------------------------------------------------------------------------------- - # Enable/disable balancer - # Returns previous balancer switch setting. - def balance_switch(enableDisable) - @admin.setBalancerRunning( - java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false)) - end - - #---------------------------------------------------------------------------------------------- - # Request a scan of the catalog table (for garbage collection) - # Returns an int signifying the number of entries cleaned - def catalogjanitor_run() - @admin.runCatalogScan() - end - - #---------------------------------------------------------------------------------------------- - # Enable/disable the catalog janitor - # Returns previous catalog janitor switch setting. - def catalogjanitor_switch(enableDisable) - @admin.enableCatalogJanitor(java.lang.Boolean::valueOf(enableDisable)) - end - - #---------------------------------------------------------------------------------------------- - # Query on the catalog janitor state (enabled/disabled?) - # Returns catalog janitor state (true signifies enabled). - def catalogjanitor_enabled() - @admin.isCatalogJanitorEnabled() - end - - #---------------------------------------------------------------------------------------------- - # Enables a table - def enable(table_name) - tableExists(table_name) - return if enabled?(table_name) - @admin.enableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Enables all tables matching the given regex - def enable_all(regex) - regex = regex.to_s - @admin.enableTables(regex) - end - - #---------------------------------------------------------------------------------------------- - # Disables a table - def disable(table_name) - tableExists(table_name) - return if disabled?(table_name) - @admin.disableTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Disables all tables matching the given regex - def disable_all(regex) - regex = regex.to_s - @admin.disableTables(regex).map { |t| t.getTableName().getNameAsString } - end - - #--------------------------------------------------------------------------------------------- - # Throw exception if table doesn't exist - def tableExists(table_name) - raise ArgumentError, "Table #{table_name} does not exist.'" unless exists?(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table disabled? - def disabled?(table_name) - @admin.isTableDisabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop(table_name) - tableExists(table_name) - raise ArgumentError, "Table #{table_name} is enabled. Disable it first.'" if enabled?(table_name) - - @admin.deleteTable(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop_all(regex) - regex = regex.to_s - failed = @admin.deleteTables(regex).map { |t| t.getTableName().getNameAsString } - return failed - end - - #---------------------------------------------------------------------------------------------- - # Returns ZooKeeper status dump - def zk_dump - org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) - end - - #---------------------------------------------------------------------------------------------- - # Creates a table - def create(table_name, *args) - # Fail if table name is not a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Flatten params array - args = args.flatten.compact - has_columns = false - - # Start defining the table - htd = org.apache.hadoop.hbase.HTableDescriptor.new(org.apache.hadoop.hbase.TableName.valueOf(table_name)) - splits = nil - # Args are either columns or splits, add them to the table definition - # TODO: add table options support - args.each do |arg| - unless arg.kind_of?(String) || arg.kind_of?(Hash) - raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") - end - - # First, handle all the cases where arg is a column family. - if arg.kind_of?(String) or arg.has_key?(NAME) - # If the arg is a string, default action is to add a column to the table. - # If arg has a name, it must also be a column descriptor. - htd.addFamily(hcd(arg, htd)) - has_columns = true - next - end - - # Get rid of the "METHOD", which is deprecated for create. - # We'll do whatever it used to do below if it's table_att. - if (method = arg.delete(METHOD)) - raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' - end - - # The hash is not a column family. Figure out what's in it. - # First, handle splits. - if arg.has_key?(SPLITS_FILE) - splits_file = arg.delete(SPLITS_FILE) - unless File.exist?(splits_file) - raise(ArgumentError, "Splits file #{splits_file} doesn't exist") - end - arg[SPLITS] = [] - File.foreach(splits_file) do |line| - arg[SPLITS].push(line.strip()) - end - end - - if arg.has_key?(SPLITS) - splits = Java::byte[][arg[SPLITS].size].new - idx = 0 - arg.delete(SPLITS).each do |split| - splits[idx] = org.apache.hadoop.hbase.util.Bytes.toBytesBinary(split) - idx = idx + 1 - end - elsif arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO) - # deprecated region pre-split API; if one of the above is specified, will be ignored. - raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS) - raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO) - raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 - num_regions = arg.delete(NUMREGIONS) - split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) - splits = split_algo.split(JInteger.valueOf(num_regions)) - end - - # Done with splits; apply formerly-table_att parameters. - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] - htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] - htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] - htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] - htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] - set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] - set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - - arg.each_key do |ignored_key| - puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ]) - end - end - - # Fail if no column families defined - raise(ArgumentError, "Table must have at least one column family") if !has_columns - - if splits.nil? - # Perform the create table call - @admin.createTable(htd) - else - # Perform the create table call - @admin.createTable(htd, splits) - end - end - - #---------------------------------------------------------------------------------------------- - # Closes a region. - # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). - # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) - def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) - @admin.closeRegion(region_name, server) - end - end - - #---------------------------------------------------------------------------------------------- - #---------------------------------------------------------------------------------------------- - # Assign a region - def assign(region_name) - @admin.assign(region_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Unassign a region - def unassign(region_name, force) - @admin.unassign(region_name.to_java_bytes, java.lang.Boolean::valueOf(force)) - end - - #---------------------------------------------------------------------------------------------- - # Move a region - def move(encoded_region_name, server = nil) - @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) - end - - #---------------------------------------------------------------------------------------------- - # Merge two regions - def merge_region(encoded_region_a_name, encoded_region_b_name, force) - @admin.mergeRegions(encoded_region_a_name.to_java_bytes, encoded_region_b_name.to_java_bytes, java.lang.Boolean::valueOf(force)) - end - - #---------------------------------------------------------------------------------------------- - # Returns table's structure description - def describe(table_name) - @admin.getTableDescriptor(table_name.to_java_bytes).to_s - end - - #---------------------------------------------------------------------------------------------- - # Truncates table (deletes all records by recreating the table) - def truncate(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - table_description = h_table.getTableDescriptor() - raise ArgumentError, "Table #{table_name} is not enabled. Enable it first.'" unless enabled?(table_name) - yield 'Disabling table...' if block_given? - @admin.disableTable(table_name) - - yield 'Dropping table...' if block_given? - @admin.deleteTable(table_name) - - yield 'Creating table...' if block_given? - @admin.createTable(table_description) - end - - #---------------------------------------------------------------------------------------------- - # Truncates table while maintaing region boundaries (deletes all records by recreating the table) - def truncate_preserve(table_name, conf = @conf) - h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) - splits = h_table.getRegionLocations().keys().map{|i| Bytes.toString(i.getStartKey)}.delete_if{|k| k == ""}.to_java :String - splits = org.apache.hadoop.hbase.util.Bytes.toByteArrays(splits) - table_description = h_table.getTableDescriptor() - yield 'Disabling table...' if block_given? - disable(table_name) - - yield 'Dropping table...' if block_given? - drop(table_name) - - yield 'Creating table with region boundaries...' if block_given? - @admin.createTable(table_description, splits) - end - - #---------------------------------------------------------------------------------------------- - # Check the status of alter command (number of regions reopened) - def alter_status(table_name) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - status = Pair.new() - begin - status = @admin.getAlterStatus(org.apache.hadoop.hbase.TableName.valueOf(table_name)) - if status.getSecond() != 0 - puts "#{status.getSecond() - status.getFirst()}/#{status.getSecond()} regions updated." - else - puts "All regions updated." - end - sleep 1 - end while status != nil && status.getFirst() != 0 - puts "Done." - end - - #---------------------------------------------------------------------------------------------- - # Change table structure or table options - def alter(table_name, wait = true, *args) - # Table name should be a string - raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) - - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - # There should be at least one argument - raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? - - # Get table descriptor - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - - # Process all args - args.each do |arg| - - - # Normalize args to support column name only alter specs - arg = { NAME => arg } if arg.kind_of?(String) - - # Normalize args to support shortcut delete syntax - arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - - # There are 3 possible options. - # 1) Column family spec. Distinguished by having a NAME and no METHOD. - method = arg.delete(METHOD) - if method == nil and arg.has_key?(NAME) - descriptor = hcd(arg, htd) - column_name = descriptor.getNameAsString - - # If column already exist, then try to alter it. Create otherwise. - if htd.hasFamily(column_name.to_java_bytes) - @admin.modifyColumn(table_name, descriptor) - else - @admin.addColumn(table_name, descriptor) - end - - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - - # We bypass descriptor when adding column families; refresh it to apply other args correctly. - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - next - end - - # 2) Method other than table_att, with some args. - name = arg.delete(NAME) - if method != nil and method != "table_att" - # Delete column family - if method == "delete" - raise(ArgumentError, "NAME parameter missing for delete method") unless name - @admin.deleteColumn(table_name, name) - # Unset table attributes - elsif method == "table_att_unset" - raise(ArgumentError, "NAME parameter missing for table_att_unset method") unless name - if (htd.getValue(name) == nil) - raise ArgumentError, "Can not find attribute: #{name}" - end - htd.remove(name) - @admin.modifyTable(table_name.to_java_bytes, htd) - # Unknown method - else - raise ArgumentError, "Unknown method: #{method}" - end - - arg.each_key do |unknown_key| - puts("Unknown argument ignored: %s" % [unknown_key]) - end - - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - - if method == "delete" - # We bypass descriptor when deleting column families; refresh it to apply other args correctly. - htd = @admin.getTableDescriptor(table_name.to_java_bytes) - end - next - end - - # 3) Some args for the table, optionally with METHOD => table_att (deprecated) - raise(ArgumentError, "NAME argument in an unexpected place") if name - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] - htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] - htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] - htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] - htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] - htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] - set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] - set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - - # set a coprocessor attribute - valid_coproc_keys = [] - if arg.kind_of?(Hash) - arg.each do |key, value| - k = String.new(key) # prepare to strip - k.strip! - - if (k =~ /coprocessor/i) - # validate coprocessor specs - v = String.new(value) - v.strip! - if !(v =~ /^([^\|]*)\|([^\|]+)\|[\s]*([\d]*)[\s]*(\|.*)?$/) - raise ArgumentError, "Coprocessor value doesn't match spec: #{v}" - end - - # generate a coprocessor ordinal by checking max id of existing cps - maxId = 0 - htd.getValues().each do |k1, v1| - attrName = org.apache.hadoop.hbase.util.Bytes.toString(k1.get()) - # a cp key is coprocessor$(\d) - if (attrName =~ /coprocessor\$(\d+)/i) - ids = attrName.scan(/coprocessor\$(\d+)/i) - maxId = ids[0][0].to_i if ids[0][0].to_i > maxId - end - end - maxId += 1 - htd.setValue(k + "\$" + maxId.to_s, value) - valid_coproc_keys << key - end - end - - valid_coproc_keys.each do |key| - arg.delete(key) - end - - @admin.modifyTable(table_name.to_java_bytes, htd) - - arg.each_key do |unknown_key| - puts("Unknown argument ignored: %s" % [unknown_key]) - end - - if wait == true - puts "Updating all regions with the new schema..." - alter_status(table_name) - end - next - end - end - end - - def status(format) - status = @admin.getClusterStatus() - if format == "detailed" - puts("version %s" % [ status.getHBaseVersion() ]) - # Put regions in transition first because usually empty - puts("%d regionsInTransition" % status.getRegionsInTransition().size()) - for k, v in status.getRegionsInTransition() - puts(" %s" % [v]) - end - master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors()) - if master_coprocs != nil - puts("master coprocessors: %s" % master_coprocs) - end - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - for name, region in status.getLoad(server).getRegionsLoad() - puts(" %s" % [ region.getNameAsString().dump ]) - puts(" %s" % [ region.toString() ]) - end - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - elsif format == "simple" - load = 0 - regions = 0 - puts("%d live servers" % [ status.getServersSize() ]) - for server in status.getServers() - puts(" %s:%d %d" % \ - [ server.getHostname(), server.getPort(), server.getStartcode() ]) - puts(" %s" % [ status.getLoad(server).toString() ]) - load += status.getLoad(server).getNumberOfRequests() - regions += status.getLoad(server).getNumberOfRegions() - end - puts("%d dead servers" % [ status.getDeadServers() ]) - for server in status.getDeadServerNames() - puts(" %s" % [ server ]) - end - puts("Aggregate load: %d, regions: %d" % [ load , regions ] ) - else - puts "#{status.getServersSize} servers, #{status.getDeadServers} dead, #{'%.4f' % status.getAverageLoad} average load" - end - end - - #---------------------------------------------------------------------------------------------- - # - # Helper methods - # - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - #---------------------------------------------------------------------------------------------- - # Is table enabled - def enabled?(table_name) - @admin.isTableEnabled(table_name) - end - - #---------------------------------------------------------------------------------------------- - #Is supplied region name is encoded region name - def closeEncodedRegion?(region_name, server) - @admin.closeRegionWithEncodedRegionName(region_name, server) - end - - #---------------------------------------------------------------------------------------------- - # Return a new HColumnDescriptor made of passed args - def hcd(arg, htd) - # String arg, single parameter constructor - return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.kind_of?(String) - - raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg.delete(NAME) - - family = htd.getFamily(name.to_java_bytes) - # create it if it's a new family - family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) - - family.setBlockCacheEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) - family.setScope(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) - family.setInMemory(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) - family.setTimeToLive(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::TTL))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) - family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) - family.setBlocksize(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) - family.setMaxVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) - family.setMinVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) - family.setValue(COMPRESSION_COMPACT, arg.delete(COMPRESSION_COMPACT)) if arg.include?(COMPRESSION_COMPACT) - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) - bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase - unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else - family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) - end - end - if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) - compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION).upcase - unless org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) - raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(" ")) - else - family.setCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) - end - end - - set_user_metadata(family, arg.delete(METADATA)) if arg[METADATA] - set_descriptor_config(family, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - - arg.each_key do |unknown_key| - puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key]) - end - - return family - end - - #---------------------------------------------------------------------------------------------- - # Enables/disables a region by name - def online(region_name, on_off) - # Open meta table - meta = org.apache.hadoop.hbase.client.HTable.new( - org.apache.hadoop.hbase.TableName::META_TABLE_NAME) - - # Read region info - # FIXME: fail gracefully if can't find the region - region_bytes = region_name.to_java_bytes - g = org.apache.hadoop.hbase.client.Get.new(region_bytes) - g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) - hri_bytes = meta.get(g).value - - # Change region status - hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) - hri.setOffline(on_off) - - # Write it back - put = org.apache.hadoop.hbase.client.Put.new(region_bytes) - put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri)) - meta.put(put) - end - # Apply user metadata to table/column descriptor - def set_user_metadata(descriptor, metadata) - raise(ArgumentError, "#{METADATA} must be a Hash type") unless metadata.kind_of?(Hash) - for k,v in metadata - v = v.to_s unless v.nil? - descriptor.setValue(k, v) - end - end - - #---------------------------------------------------------------------------------------------- - # Take a snapshot of specified table - def snapshot(table, snapshot_name) - @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Restore specified snapshot - def restore_snapshot(snapshot_name) - @admin.restoreSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Create a new table by cloning the snapshot content - def clone_snapshot(snapshot_name, table) - @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Rename specified snapshot - def rename_snapshot(old_snapshot_name, new_snapshot_name) - @admin.renameSnapshot(old_snapshot_name.to_java_bytes, new_snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Delete specified snapshot - def delete_snapshot(snapshot_name) - @admin.deleteSnapshot(snapshot_name.to_java_bytes) - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of snapshots - def list_snapshot - @admin.listSnapshots - end - - # Apply config specific to a table/column to its descriptor - def set_descriptor_config(descriptor, config) - raise(ArgumentError, "#{CONFIGURATION} must be a Hash type") unless config.kind_of?(Hash) - for k,v in config - v = v.to_s unless v.nil? - descriptor.setConfiguration(k, v) - end - end - - #---------------------------------------------------------------------------------------------- - # Returns namespace's structure description - def describe_namespace(namespace_name) - namespace = @admin.getNamespaceDescriptor(namespace_name) - - unless namespace.nil? - return namespace.to_s - end - - raise(ArgumentError, "Failed to find namespace named #{namespace_name}") - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of namespaces in hbase - def list_namespace - @admin.listNamespaceDescriptors.map { |ns| ns.getName } - end - - #---------------------------------------------------------------------------------------------- - # Returns a list of tables in namespace - def list_namespace_tables(namespace_name) - unless namespace_name.nil? - return @admin.listTableNamesByNamespace(namespace_name).map { |t| t.getQualifierAsString() } - end - - raise(ArgumentError, "Failed to find namespace named #{namespace_name}") - end - - #---------------------------------------------------------------------------------------------- - # Creates a namespace - def create_namespace(namespace_name, *args) - # Fail if table name is not a string - raise(ArgumentError, "Namespace name must be of type String") unless namespace_name.kind_of?(String) - - # Flatten params array - args = args.flatten.compact - - # Start defining the table - nsb = org.apache.hadoop.hbase.NamespaceDescriptor::create(namespace_name) - args.each do |arg| - unless arg.kind_of?(Hash) - raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") - end - for k,v in arg - v = v.to_s unless v.nil? - nsb.addConfiguration(k, v) - end - end - @admin.createNamespace(nsb.build()); - end - - #---------------------------------------------------------------------------------------------- - # modify a namespace - def alter_namespace(namespace_name, *args) - # Fail if table name is not a string - raise(ArgumentError, "Namespace name must be of type String") unless namespace_name.kind_of?(String) - - nsd = @admin.getNamespaceDescriptor(namespace_name) - - unless nsd - raise(ArgumentError, "Namespace does not exist") - end - nsb = org.apache.hadoop.hbase.NamespaceDescriptor::create(nsd) - - # Flatten params array - args = args.flatten.compact - - # Start defining the table - args.each do |arg| - unless arg.kind_of?(Hash) - raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash type") - end - method = arg[METHOD] - if method == "unset" - nsb.removeConfiguration(arg[NAME]) - elsif method == "set" - arg.delete(METHOD) - for k,v in arg - v = v.to_s unless v.nil? - - nsb.addConfiguration(k, v) - end - else - raise(ArgumentError, "Unknown method #{method}") - end - end - @admin.modifyNamespace(nsb.build()); - end - - - #---------------------------------------------------------------------------------------------- - # Drops a table - def drop_namespace(namespace_name) - @admin.deleteNamespace(namespace_name) - end - - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb deleted file mode 100644 index 64482c5e0d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/hbase.rb +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -require 'hbase/admin' -require 'hbase/table' -require 'hbase/security' - -module Hbase - class Hbase - attr_accessor :configuration - - def initialize(config = nil) - # Create configuration - if config - self.configuration = config - else - self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create - # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. - configuration.setInt("hbase.client.retries.number", 7) - configuration.setInt("ipc.client.connect.max.retries", 3) - end - end - - def admin(formatter) - ::Hbase::Admin.new(configuration, formatter) - end - - # Create new one each time - def table(table, shell) - ::Hbase::Table.new(configuration, table, shell) - end - - def replication_admin(formatter) - ::Hbase::RepAdmin.new(configuration, formatter) - end - - def security_admin(formatter) - ::Hbase::SecurityAdmin.new(configuration, formatter) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb deleted file mode 100644 index baa6aeaeda..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/replication_admin.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class RepAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @replication_admin = org.apache.hadoop.hbase.client.replication.ReplicationAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - # Add a new peer cluster to replicate to - def add_peer(id, cluster_key) - @replication_admin.addPeer(id, cluster_key) - end - - #---------------------------------------------------------------------------------------------- - # Remove a peer cluster, stops the replication - def remove_peer(id) - @replication_admin.removePeer(id) - end - - - #--------------------------------------------------------------------------------------------- - # Show replcated tables/column families, and their ReplicationType - def list_replicated_tables - @replication_admin.listReplicated() - end - - #---------------------------------------------------------------------------------------------- - # List all peer clusters - def list_peers - @replication_admin.listPeers - end - - #---------------------------------------------------------------------------------------------- - # Get peer cluster state - def get_peer_state(id) - @replication_admin.getPeerState(id) ? "ENABLED" : "DISABLED" - end - - #---------------------------------------------------------------------------------------------- - # Restart the replication stream to the specified peer - def enable_peer(id) - @replication_admin.enablePeer(id) - end - - #---------------------------------------------------------------------------------------------- - # Stop the replication stream to the specified peer - def disable_peer(id) - @replication_admin.disablePeer(id) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb deleted file mode 100644 index e370ca9671..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/security.rb +++ /dev/null @@ -1,233 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin - -module Hbase - class SecurityAdmin - include HBaseConstants - - def initialize(configuration, formatter) - @config = configuration - @admin = org.apache.hadoop.hbase.client.HBaseAdmin.new(configuration) - @formatter = formatter - end - - #---------------------------------------------------------------------------------------------- - def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - perm = org.apache.hadoop.hbase.security.access.Permission.new( - permissions.to_java_bytes) - - # Verify that the specified permission is valid - if (permissions == nil || permissions.length == 0) - raise(ArgumentError, "Invalid permission: no actions associated with user") - end - - if (table_name != nil) - tablebytes=table_name.to_java_bytes - #check if the tablename passed is actually a namespace - if (isNamespace?(table_name)) - # Namespace should exist first. - namespace_name = table_name[1...table_name.length] - raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) - - #We pass the namespace name along with "@" so that we can differentiate a namespace from a table. - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, tablebytes, perm.getActions()) - else - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name.to_java_bytes) - htd = @admin.getTableDescriptor(tablebytes) - - if (family != nil) - raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, tableName, fambytes, - qualbytes, perm.getActions()) - end - else - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.grant( - protocol, user, perm.getActions()) - end - - ensure - meta_table.close() - end - end - - #---------------------------------------------------------------------------------------------- - def revoke(user, table_name=nil, family=nil, qualifier=nil) - security_available? - - # TODO: need to validate user name - - begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - - if (table_name != nil) - #check if the tablename passed is actually a namespace - if (isNamespace?(table_name)) - # Namespace should exist first. - namespace_name = table_name[1...table_name.length] - raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) - - #We pass the namespace name along with "@" so that we can differentiate a namespace from a table. - tablebytes=table_name.to_java_bytes - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke( - protocol, user, tablebytes) - else - # Table should exist - raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) - - tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name.to_java_bytes) - htd = @admin.getTableDescriptor(tableName) - - if (family != nil) - raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) - end - - fambytes = family.to_java_bytes if (family != nil) - qualbytes = qualifier.to_java_bytes if (qualifier != nil) - - # invoke cp endpoint to perform access controlse - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke( - protocol, user, tableName, fambytes, qualbytes) - end - else - # invoke cp endpoint to perform access controlse - perm = org.apache.hadoop.hbase.security.access.Permission.new(''.to_java_bytes) - org.apache.hadoop.hbase.protobuf.ProtobufUtil.revoke(protocol, user, perm.getActions()) - end - ensure - meta_table.close() - end - end - - #---------------------------------------------------------------------------------------------- - def user_permission(table_name=nil) - security_available? - - begin - meta_table = org.apache.hadoop.hbase.client.HTable.new(@config, - org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - service = meta_table.coprocessorService( - org.apache.hadoop.hbase.HConstants::EMPTY_START_ROW) - - protocol = org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos:: - AccessControlService.newBlockingStub(service) - - if (table_name != nil) - #check if namespace is passed. - if (isNamespace?(table_name)) - # Namespace should exist first. - namespace_name = table_name[1...table_name.length] - raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) - # invoke cp endpoint to perform access controls - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions( - protocol, table_name.to_java_bytes) - else - raise(ArgumentError, "Can't find table: #{table_name}") unless exists?(table_name) - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions( - protocol, org.apache.hadoop.hbase.TableName.valueOf(table_name)) - end - else - perms = org.apache.hadoop.hbase.protobuf.ProtobufUtil.getUserPermissions(protocol) - end - ensure - meta_table.close() - end - - res = {} - count = 0 - perms.each do |value| - user_name = String.from_java_bytes(value.getUser) - table = (value.getTable != nil) ? value.getTable.toString() : '' - family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' - qualifier = (value.getQualifier != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : '' - - action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions - - if block_given? - yield(user_name, "#{table},#{family},#{qualifier}: #{action.to_s}") - else - res[user_name] ||= {} - res[user_name][family + ":" +qualifier] = action - end - count += 1 - end - - return ((block_given?) ? count : res) - end - - # Does table exist? - def exists?(table_name) - @admin.tableExists(table_name) - end - - def isNamespace?(table_name) - table_name.start_with?('@') - end - - # Does Namespace exist - def namespace_exists?(namespace_name) - namespaceDesc = @admin.getNamespaceDescriptor(namespace_name) - if(namespaceDesc == nil) - return false - else - return true - end - end - - # Make sure that security tables are available - def security_available?() - raise(ArgumentError, "DISABLED: Security features are not available") \ - unless exists?(org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb deleted file mode 100644 index 13d7646be3..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/hbase/table.rb +++ /dev/null @@ -1,539 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include Java - -# Wrapper for org.apache.hadoop.hbase.client.HTable - -module Hbase - class Table - include HBaseConstants - - @@thread_pool = nil - - # Add the command 'name' to table s.t. the shell command also called via 'name' - # and has an internal method also called 'name'. - # - # e.g. name = scan, adds table.scan which calls Scan.scan - def self.add_shell_command(name) - self.add_command(name, name, name) - end - - # add a named command to the table instance - # - # name - name of the command that should added to the table - # (eg. sending 'scan' here would allow you to do table.scan) - # shell_command - name of the command in the shell - # internal_method_name - name of the method in the shell command to forward the call - def self.add_command(name, shell_command, internal_method_name) - method = name.to_sym - self.class_eval do - define_method method do |*args| - @shell.internal_command(shell_command, internal_method_name, self, *args) - end - end - end - - # General help for the table - # class level so we can call it from anywhere - def self.help - return <<-EOF -Help for table-reference commands. - -You can either create a table via 'create' and then manipulate the table via commands like 'put', 'get', etc. -See the standard help information for how to use each of these commands. - -However, as of 0.96, you can also get a reference to a table, on which you can invoke commands. -For instance, you can get create a table and keep around a reference to it via: - - hbase> t = create 't', 'cf' - -Or, if you have already created the table, you can get a reference to it: - - hbase> t = get_table 't' - -You can do things like call 'put' on the table: - - hbase> t.put 'r', 'cf:q', 'v' - -which puts a row 'r' with column family 'cf', qualifier 'q' and value 'v' into table t. - -To read the data out, you can scan the table: - - hbase> t.scan - -which will read all the rows in table 't'. - -Essentially, any command that takes a table name can also be done via table reference. -Other commands include things like: get, delete, deleteall, -get_all_columns, get_counter, count, incr. These functions, along with -the standard JRuby object methods are also available via tab completion. - -For more information on how to use each of these commands, you can also just type: - - hbase> t.help 'scan' - -which will output more information on how to use that command. - -You can also do general admin actions directly on a table; things like enable, disable, -flush and drop just by typing: - - hbase> t.enable - hbase> t.flush - hbase> t.disable - hbase> t.drop - -Note that after dropping a table, your reference to it becomes useless and further usage -is undefined (and not recommended). -EOF - end - - #--------------------------------------------------------------------------------------------- - - # let external objects read the underlying table object - attr_reader :table - # let external objects read the table name - attr_reader :name - - def initialize(configuration, table_name, shell) - if @@thread_pool then - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name.to_java_bytes, @@thread_pool) - else - @table = org.apache.hadoop.hbase.client.HTable.new(configuration, table_name) - @@thread_pool = @table.getPool() - end - @name = table_name - @shell = shell - @converters = Hash.new() - end - - # Note the below methods are prefixed with '_' to hide them from the average user, as - # they will be much less likely to tab complete to the 'dangerous' internal method - #---------------------------------------------------------------------------------------------- - # Put a cell 'value' at specified table/row/column - def _put_internal(row, column, value, timestamp = nil) - p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) - family, qualifier = parse_column_name(column) - if timestamp - p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) - else - p.add(family, qualifier, value.to_s.to_java_bytes) - end - @table.put(p) - end - - #---------------------------------------------------------------------------------------------- - # Delete a cell - def _delete_internal(row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - _deleteall_internal(row, column, timestamp) - end - - #---------------------------------------------------------------------------------------------- - # Delete a row - def _deleteall_internal(row, column = nil, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - raise ArgumentError, "Row Not Found" if _get_internal(row).nil? - d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp) - if column - family, qualifier = parse_column_name(column) - d.deleteColumns(family, qualifier, timestamp) - end - @table.delete(d) - end - - #---------------------------------------------------------------------------------------------- - # Increment a counter atomically - def _incr_internal(row, column, value = nil) - value ||= 1 - family, qualifier = parse_column_name(column) - if qualifier.nil? - raise ArgumentError, "Failed to provide both column family and column qualifier for incr" - end - @table.incrementColumnValue(row.to_s.to_java_bytes, family, qualifier, value) - end - - #---------------------------------------------------------------------------------------------- - # Count rows in a table - def _count_internal(interval = 1000, caching_rows = 10) - # We can safely set scanner caching with the first key only filter - scan = org.apache.hadoop.hbase.client.Scan.new - scan.cache_blocks = false - scan.caching = caching_rows - scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) - - # Run the scanner - scanner = @table.getScanner(scan) - count = 0 - iter = scanner.iterator - - # Iterate results - while iter.hasNext - row = iter.next - count += 1 - next unless (block_given? && count % interval == 0) - # Allow command modules to visualize counting process - yield(count, - org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) - end - - # Return the counter - return count - end - - #---------------------------------------------------------------------------------------------- - # Get from table - def _get_internal(row, *args) - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - maxlength = -1 - @converters.clear() - - # Normalize args - args = args.first if args.first.kind_of?(Hash) - if args.kind_of?(String) || args.kind_of?(Array) - columns = [ args ].flatten.compact - args = { COLUMNS => columns } - end - - # - # Parse arguments - # - unless args.kind_of?(Hash) - raise ArgumentError, "Failed parse of of #{args.inspect}, #{args.class}" - end - - # Get maxlength parameter if passed - maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] - filter = args.delete(FILTER) if args[FILTER] - - unless args.empty? - columns = args[COLUMN] || args[COLUMNS] - if args[VERSIONS] - vers = args[VERSIONS] - else - vers = 1 - end - if columns - # Normalize types, convert string to an array of strings - columns = [ columns ] if columns.is_a?(String) - - # At this point it is either an array or some unsupported stuff - unless columns.kind_of?(Array) - raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" - end - - # Get each column name and add it to the filter - columns.each do |column| - family, qualifier = parse_column_name(column.to_s) - if qualifier - get.addColumn(family, qualifier) - else - get.addFamily(family) - end - end - - # Additional params - get.setMaxVersions(vers) - get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - else - # May have passed TIMESTAMP and row only; wants all columns from ts. - unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] - raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" - end - - get.setMaxVersions(vers) - # Set the timestamp/timerange - get.setTimeStamp(ts.to_i) if args[TIMESTAMP] - get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] - end - end - - unless filter.class == String - get.setFilter(filter) - else - get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - # Call hbase for the results - result = @table.get(get) - return nil if result.isEmpty - - # Print out results. Result can be Cell or RowResult. - res = {} - result.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - value = to_string(column, kv, maxlength) - - if block_given? - yield(column, value) - else - res[column] = value - end - end - - # If block given, we've yielded all the results, otherwise just return them - return ((block_given?) ? nil : res) - end - - #---------------------------------------------------------------------------------------------- - # Fetches and decodes a counter value from hbase - def _get_counter_internal(row, column) - family, qualifier = parse_column_name(column.to_s) - # Format get request - get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) - get.addColumn(family, qualifier) - get.setMaxVersions(1) - - # Call hbase - result = @table.get(get) - return nil if result.isEmpty - - # Fetch cell value - cell = result.list[0] - org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) - end - - #---------------------------------------------------------------------------------------------- - # Scans whole table or a range of keys and returns rows matching specific criteria - def _scan_internal(args = {}) - unless args.kind_of?(Hash) - raise ArgumentError, "Arguments should be a hash. Failed to parse #{args.inspect}, #{args.class}" - end - - limit = args.delete("LIMIT") || -1 - maxlength = args.delete("MAXLENGTH") || -1 - @converters.clear() - - if args.any? - filter = args["FILTER"] - startrow = args["STARTROW"] || '' - stoprow = args["STOPROW"] - timestamp = args["TIMESTAMP"] - columns = args["COLUMNS"] || args["COLUMN"] || [] - cache_blocks = args["CACHE_BLOCKS"] || true - cache = args["CACHE"] || 0 - versions = args["VERSIONS"] || 1 - timerange = args[TIMERANGE] - raw = args["RAW"] || false - - # Normalize column names - columns = [columns] if columns.class == String - unless columns.kind_of?(Array) - raise ArgumentError.new("COLUMNS must be specified as a String or an Array") - end - - scan = if stoprow - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) - else - org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) - end - - columns.each do |c| - family, qualifier = parse_column_name(c.to_s) - if qualifier - scan.addColumn(family, qualifier) - else - scan.addFamily(family) - end - end - - unless filter.class == String - scan.setFilter(filter) - else - scan.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) - end - - scan.setTimeStamp(timestamp) if timestamp - scan.setCacheBlocks(cache_blocks) - scan.setCaching(cache) if cache > 0 - scan.setMaxVersions(versions) if versions > 1 - scan.setTimeRange(timerange[0], timerange[1]) if timerange - scan.setRaw(raw) - else - scan = org.apache.hadoop.hbase.client.Scan.new - end - - # Start the scanner - scanner = @table.getScanner(scan) - count = 0 - res = {} - iter = scanner.iterator - - # Iterate results - while iter.hasNext - if limit > 0 && count >= limit - break - end - - row = iter.next - key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) - - row.list.each do |kv| - family = String.from_java_bytes(kv.getFamily) - qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) - - column = "#{family}:#{qualifier}" - cell = to_string(column, kv, maxlength) - - if block_given? - yield(key, "column=#{column}, #{cell}") - else - res[key] ||= {} - res[key][column] = cell - end - end - - # One more row processed - count += 1 - end - - return ((block_given?) ? count : res) - end - - #---------------------------- - # Add general administration utilities to the shell - # each of the names below adds this method name to the table - # by callling the corresponding method in the shell - # Add single method utilities to the current class - # Generally used for admin functions which just have one name and take the table name - def self.add_admin_utils(*args) - args.each do |method| - define_method method do |*method_args| - @shell.command(method, @name, *method_args) - end - end - end - - #Add the following admin utilities to the table - add_admin_utils :enable, :disable, :flush, :drop, :describe, :snapshot - - #---------------------------- - #give the general help for the table - # or the named command - def help (command = nil) - #if there is a command, get the per-command help from the shell - if command - begin - return @shell.help_command(command) - rescue NoMethodError - puts "Command \'#{command}\' does not exist. Please see general table help." - return nil - end - end - return @shell.help('table_help') - end - - # Table to string - def to_s - cl = self.class() - return "#{cl} - #{@name}" - end - - # Standard ruby call to get the return value for an object - # overriden here so we get sane semantics for printing a table on return - def inspect - to_s - end - - #---------------------------------------------------------------------------------------- - # Helper methods - - # Returns a list of column names in the table - def get_all_columns - @table.table_descriptor.getFamilies.map do |family| - "#{family.getNameAsString}:" - end - end - - # Checks if current table is one of the 'meta' tables - def is_meta_table? - tn = @table.table_name - org.apache.hadoop.hbase.util.Bytes.equals(tn, - org.apache.hadoop.hbase.TableName::META_TABLE_NAME.getName) - end - - # Returns family and (when has it) qualifier for a column name - def parse_column_name(column) - split = org.apache.hadoop.hbase.KeyValue.parseColumn(column.to_java_bytes) - set_converter(split) if split.length > 1 - return split[0], (split.length > 1) ? split[1] : nil - end - - # Make a String of the passed kv - # Intercept cells whose format we know such as the info:regioninfo in hbase:meta - def to_string(column, kv, maxlength = -1) - if is_meta_table? - if column == 'info:regioninfo' or column == 'info:splitA' or column == 'info:splitB' - hri = org.apache.hadoop.hbase.HRegionInfo.parseFromOrNull(kv.getValue) - return "timestamp=%d, value=%s" % [kv.getTimestamp, hri.toString] - end - if column == 'info:serverstartcode' - if kv.getValue.length > 0 - str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValue) - else - str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValue) - end - return "timestamp=%d, value=%s" % [kv.getTimestamp, str_val] - end - end - - if kv.isDelete - val = "timestamp=#{kv.getTimestamp}, type=#{org.apache.hadoop.hbase.KeyValue::Type::codeToType(kv.getType)}" - else - val = "timestamp=#{kv.getTimestamp}, value=#{convert(column, kv)}" - end - (maxlength != -1) ? val[0, maxlength] : val - end - - def convert(column, kv) - #use org.apache.hadoop.hbase.util.Bytes as the default class - klazz_name = 'org.apache.hadoop.hbase.util.Bytes' - #use org.apache.hadoop.hbase.util.Bytes::toStringBinary as the default convertor - converter = 'toStringBinary' - if @converters.has_key?(column) - # lookup the CONVERTER for certain column - "cf:qualifier" - matches = /c\((.+)\)\.(.+)/.match(@converters[column]) - if matches.nil? - # cannot match the pattern of 'c(className).functionname' - # use the default klazz_name - converter = @converters[column] - else - klazz_name = matches[1] - converter = matches[2] - end - end - method = eval(klazz_name).method(converter) - return method.call(kv.getValue) # apply the converter - end - - # if the column spec contains CONVERTER information, to get rid of :CONVERTER info from column pair. - # 1. return back normal column pair as usual, i.e., "cf:qualifier[:CONVERTER]" to "cf" and "qualifier" only - # 2. register the CONVERTER information based on column spec - "cf:qualifier" - def set_converter(column) - family = String.from_java_bytes(column[0]) - parts = org.apache.hadoop.hbase.KeyValue.parseColumn(column[1]) - if parts.length > 1 - @converters["#{family}:#{String.from_java_bytes(parts[0])}"] = String.from_java_bytes(parts[1]) - column[1] = parts[0] - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb deleted file mode 100644 index b32e691cf8..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/irb/hirb.rb +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -require 'rbconfig' - -module IRB - WINDOZE = Config::CONFIG['host_os'] =~ /mswin|mingw/ - - # Subclass of IRB so can intercept methods - class HIRB < Irb - def initialize - # This is ugly. Our 'help' method above provokes the following message - # on irb construction: 'irb: warn: can't alias help from irb_help.' - # Below, we reset the output so its pointed at /dev/null during irb - # construction just so this message does not come out after we emit - # the banner. Other attempts at playing with the hash of methods - # down in IRB didn't seem to work. I think the worst thing that can - # happen is the shell exiting because of failed IRB construction with - # no error (though we're not blanking STDERR) - begin - # Map the '/dev/null' according to the runing platform - # Under Windows platform the 'dev/null' is not fully compliant with unix, - # and the 'NUL' object need to be use instead. - devnull = "/dev/null" - devnull = "NUL" if WINDOZE - f = File.open(devnull, "w") - $stdout = f - super - ensure - f.close() - $stdout = STDOUT - end - end - - def output_value - # Suppress output if last_value is 'nil' - # Otherwise, when user types help, get ugly 'nil' - # after all output. - if @context.last_value != nil - super - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb deleted file mode 100644 index 7c1ed17812..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell.rb +++ /dev/null @@ -1,348 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Shell commands module -module Shell - @@commands = {} - def self.commands - @@commands - end - - @@command_groups = {} - def self.command_groups - @@command_groups - end - - def self.load_command(name, group) - return if commands[name] - - # Register command in the group - raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] - command_groups[group][:commands] << name - - # Load command - begin - require "shell/commands/#{name}" - klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize - commands[name] = eval("Commands::#{klass_name}") - rescue => e - raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" - end - end - - def self.load_command_group(group, opts) - raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] - - command_groups[group] = { - :commands => [], - :command_names => opts[:commands], - :full_name => opts[:full_name] || group, - :comment => opts[:comment] - } - - opts[:commands].each do |command| - load_command(command, group) - end - end - - #---------------------------------------------------------------------- - class Shell - attr_accessor :hbase - attr_accessor :formatter - - @debug = false - attr_accessor :debug - - def initialize(hbase, formatter) - self.hbase = hbase - self.formatter = formatter - end - - def hbase_admin - @hbase_admin ||= hbase.admin(formatter) - end - - def hbase_table(name) - hbase.table(name, self) - end - - def hbase_replication_admin - @hbase_replication_admin ||= hbase.replication_admin(formatter) - end - - def hbase_security_admin - @hbase_security_admin ||= hbase.security_admin(formatter) - end - - def export_commands(where) - ::Shell.commands.keys.each do |cmd| - # here where is the IRB namespace - # this method just adds the call to the specified command - # which just references back to 'this' shell object - # a decently extensible way to add commands - where.send :instance_eval, <<-EOF - def #{cmd}(*args) - ret = @shell.command('#{cmd}', *args) - puts - return ret - end - EOF - end - end - - def command_instance(command) - ::Shell.commands[command.to_s].new(self) - end - - #call the method 'command' on the specified command - def command(command, *args) - internal_command(command, :command, *args) - end - - #call a specific internal method in the command instance - # command - name of the command to call - # method_name - name of the method on the command to call. Defaults to just 'command' - # args - to be passed to the named method - def internal_command(command, method_name= :command, *args) - command_instance(command).command_safe(self.debug,method_name, *args) - end - - def print_banner - puts "HBase Shell; enter 'help' for list of supported commands." - puts 'Type "exit" to leave the HBase Shell' - print 'Version ' - command('version') - puts - end - - def help_multi_command(command) - puts "Command: #{command}" - puts command_instance(command).help - puts - return nil - end - - def help_command(command) - puts command_instance(command).help - return nil - end - - def help_group(group_name) - group = ::Shell.command_groups[group_name.to_s] - group[:commands].sort.each { |cmd| help_multi_command(cmd) } - if group[:comment] - puts '-' * 80 - puts - puts group[:comment] - puts - end - return nil - end - - def help(command = nil) - if command - return help_command(command) if ::Shell.commands[command.to_s] - return help_group(command) if ::Shell.command_groups[command.to_s] - puts "ERROR: Invalid command or command group name: #{command}" - puts - end - - puts help_header - puts - puts 'COMMAND GROUPS:' - ::Shell.command_groups.each do |name, group| - puts " Group name: " + name - puts " Commands: " + group[:command_names].sort.join(', ') - puts - end - unless command - puts 'SHELL USAGE:' - help_footer - end - return nil - end - - def help_header - return "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + "\n" + - "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" + - "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." - end - - def help_footer - puts <<-HERE -Quote all names in HBase Shell such as table and column names. Commas delimit -command parameters. Type after entering a command to run it. -Dictionaries of configuration used in the creation and alteration of tables are -Ruby Hashes. They look like this: - - {'key1' => 'value1', 'key2' => 'value2', ...} - -and are opened and closed with curley-braces. Key/values are delimited by the -'=>' character combination. Usually keys are predefined constants such as -NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type -'Object.constants' to see a (messy) list of all constants in the environment. - -If you are using binary keys or values and need to enter them in the shell, use -double-quote'd hexadecimal representation. For example: - - hbase> get 't1', "key\\x03\\x3f\\xcd" - hbase> get 't1', "key\\003\\023\\011" - hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" - -The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. -For more on the HBase Shell, see http://hbase.apache.org/docs/current/book.html - HERE - end - end -end - -# Load commands base class -require 'shell/commands' - -# Load all commands -Shell.load_command_group( - 'general', - :full_name => 'GENERAL HBASE SHELL COMMANDS', - :commands => %w[ - status - version - table_help - whoami - ] -) - -Shell.load_command_group( - 'ddl', - :full_name => 'TABLES MANAGEMENT COMMANDS', - :commands => %w[ - alter - create - describe - disable - disable_all - is_disabled - drop - drop_all - enable - enable_all - is_enabled - exists - list - show_filters - alter_status - alter_async - get_table - ] -) - -Shell.load_command_group( - 'namespace', - :full_name => 'NAMESPACE MANAGEMENT COMMANDS', - :commands => %w[ - create_namespace - drop_namespace - alter_namespace - describe_namespace - list_namespace - list_namespace_tables - ] -) - -Shell.load_command_group( - 'dml', - :full_name => 'DATA MANIPULATION COMMANDS', - :commands => %w[ - count - delete - deleteall - get - get_counter - incr - put - scan - truncate - truncate_preserve - ] -) - -Shell.load_command_group( - 'tools', - :full_name => 'HBASE SURGERY TOOLS', - :comment => "WARNING: Above commands are for 'experts'-only as misuse can damage an install", - :commands => %w[ - assign - balancer - balance_switch - close_region - compact - flush - major_compact - move - split - merge_region - unassign - zk_dump - hlog_roll - catalogjanitor_run - catalogjanitor_switch - catalogjanitor_enabled - trace - ] -) - -Shell.load_command_group( - 'replication', - :full_name => 'CLUSTER REPLICATION TOOLS', - :comment => "In order to use these tools, hbase.replication must be true.", - :commands => %w[ - add_peer - remove_peer - list_peers - enable_peer - disable_peer - list_replicated_tables - ] -) - -Shell.load_command_group( - 'snapshot', - :full_name => 'CLUSTER SNAPSHOT TOOLS', - :commands => %w[ - snapshot - clone_snapshot - restore_snapshot - rename_snapshot - delete_snapshot - list_snapshots - ] -) - -Shell.load_command_group( - 'security', - :full_name => 'SECURITY TOOLS', - :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", - :commands => %w[ - grant - revoke - user_permission - ] -) - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb deleted file mode 100644 index 4303aa6e45..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands.rb +++ /dev/null @@ -1,120 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Command - - def initialize(shell) - @shell = shell - end - - #wrap an execution of cmd to catch hbase exceptions - # cmd - command name to execture - # args - arguments to pass to the command - def command_safe(debug, cmd = :command, *args) - # send is internal ruby method to call 'cmd' with *args - #(everything is a message, so this is just the formal semantics to support that idiom) - translate_hbase_exceptions(*args) { send(cmd,*args) } - rescue => e - rootCause = e - while rootCause != nil && rootCause.respond_to?(:cause) && rootCause.cause != nil - rootCause = rootCause.cause - end - puts - puts "ERROR: #{rootCause}" - puts "Backtrace: #{rootCause.backtrace.join("\n ")}" if debug - puts - puts "Here is some help for this command:" - puts help - puts - end - - def admin - @shell.hbase_admin - end - - def table(name) - @shell.hbase_table(name) - end - - def replication_admin - @shell.hbase_replication_admin - end - - def security_admin - @shell.hbase_security_admin - end - - #---------------------------------------------------------------------- - - def formatter - @shell.formatter - end - - def format_simple_command - now = Time.now - yield - formatter.header - formatter.footer(now) - end - - def format_and_return_simple_command - now = Time.now - ret = yield - formatter.header - formatter.footer(now) - return ret - end - - def translate_hbase_exceptions(*args) - yield - rescue => e - raise e unless e.respond_to?(:cause) && e.cause != nil - - # Get the special java exception which will be handled - cause = e.cause - if cause.kind_of?(org.apache.hadoop.hbase.TableNotFoundException) then - raise "Unknown table #{args.first}!" - end - if cause.kind_of?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) then - valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } - raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" - end - if cause.kind_of?(org.apache.hadoop.hbase.TableExistsException) then - raise "Table already exists: #{args.first}!" - end - # To be safe, here only AccessDeniedException is considered. In future - # we might support more in more generic approach when possible. - if cause.kind_of?(org.apache.hadoop.hbase.security.AccessDeniedException) then - str = java.lang.String.new("#{cause}") - # Error message is merged with stack trace, reference StringUtils.stringifyException - # This is to parse and get the error message from the whole. - strs = str.split("\n") - if strs.size > 0 then - raise "#{strs[0]}" - end - end - - # Throw the other exception which hasn't been handled above - raise e - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb deleted file mode 100644 index deb5c5def5..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/add_peer.rb +++ /dev/null @@ -1,43 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AddPeer< Command - def help - return <<-EOF -Add a peer cluster to replicate to, the id must be a short and -the cluster key is composed like this: -hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent -This gives a full path for HBase to connect to another cluster. -Examples: - - hbase> add_peer '1', "server1.cie.com:2181:/hbase" - hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" -EOF - end - - def command(id, cluster_key) - format_simple_command do - replication_admin.add_peer(id, cluster_key) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb deleted file mode 100644 index f22348ce44..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter.rb +++ /dev/null @@ -1,92 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Alter < Command - def help - return <<-EOF -Alter a table. Depending on the HBase setting ("hbase.online.schema.update.enable"), -the table must be disabled or not to be altered (see help 'disable'). -You can add/modify/delete column families, as well as change table -configuration. Column families work similarly to create; column family -spec can either be a name string, or a dictionary with NAME attribute. -Dictionaries are described on the main help command output. - -For example, to change or add the 'f1' column family in table 't1' from -current value to keep a maximum of 5 cell VERSIONS, do: - - hbase> alter 't1', NAME => 'f1', VERSIONS => 5 - -You can operate on several column families: - - hbase> alter 't1', 'f1', {NAME => 'f2', IN_MEMORY => true}, {NAME => 'f3', VERSIONS => 5} - -To delete the 'f1' column family in table 't1', use one of: - - hbase> alter 't1', NAME => 'f1', METHOD => 'delete' - hbase> alter 't1', 'delete' => 'f1' - -You can also change table-scope attributes like MAX_FILESIZE, READONLY, -MEMSTORE_FLUSHSIZE, DEFERRED_LOG_FLUSH, etc. These can be put at the end; -for example, to change the max size of a region to 128MB, do: - - hbase> alter 't1', MAX_FILESIZE => '134217728' - -You can add a table coprocessor by setting a table coprocessor attribute: - - hbase> alter 't1', - 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' - -Since you can have multiple coprocessors configured for a table, a -sequence number will be automatically appended to the attribute name -to uniquely identify it. - -The coprocessor attribute must match the pattern below in order for -the framework to understand how to load the coprocessor classes: - - [coprocessor jar file location] | class name | [priority] | [arguments] - -You can also set configuration settings specific to this table or column family: - - hbase> alter 't1', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'} - hbase> alter 't1', {NAME => 'f2', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} - -You can also remove a table-scope attribute: - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' - - hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' - -There could be more than one alteration in one command: - - hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, - { MAX_FILESIZE => '134217728' }, { METHOD => 'delete', NAME => 'f2' }, - OWNER => 'johndoe', METADATA => { 'mykey' => 'myvalue' } -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, true, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb deleted file mode 100644 index 164f339149..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_async.rb +++ /dev/null @@ -1,65 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterAsync < Command - def help - return <<-EOF -Alter column family schema, does not wait for all regions to receive the -schema changes. Pass table name and a dictionary specifying new column -family schema. Dictionaries are described on the main help command output. -Dictionary must include name of column family to alter. For example, - -To change or add the 'f1' column family in table 't1' from defaults -to instead keep a maximum of 5 cell VERSIONS, do: - - hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 - -To delete the 'f1' column family in table 't1', do: - - hbase> alter_async 't1', NAME => 'f1', METHOD => 'delete' - -or a shorter version: - - hbase> alter_async 't1', 'delete' => 'f1' - -You can also change table-scope attributes like MAX_FILESIZE -MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. - -For example, to change the max size of a family to 128MB, do: - - hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' - -There could be more than one alteration in one command: - - hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} - -To check if all the regions have been updated, use alter_status -EOF - end - - def command(table, *args) - format_simple_command do - admin.alter(table, false, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb deleted file mode 100644 index 760bbf7bc5..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_namespace.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterNamespace < Command - def help - return <<-EOF -Alter namespace properties. - -To add/modify a property: - - hbase> alter_namespace 'ns1', {METHOD => 'set', 'PROERTY_NAME' => 'PROPERTY_VALUE'} - -To delete a property: - - hbase> alter_namespace 'ns1', {METHOD => 'unset', NAME=>'PROERTY_NAME'} -EOF - end - - def command(namespace, *args) - format_simple_command do - admin.alter_namespace(namespace, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb deleted file mode 100644 index d62edacc9c..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/alter_status.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class AlterStatus < Command - def help - return <<-EOF -Get the status of the alter command. Indicates the number of regions of the -table that have received the updated schema -Pass table name. - -hbase> alter_status 't1' -EOF - end - def command(table) - admin.alter_status(table) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb deleted file mode 100644 index 448a5460b4..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/assign.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Assign < Command - def help - return <<-EOF -Assign a region. Use with caution. If region already assigned, -this command will do a force reassign. For experts only. -Examples: - - hbase> assign 'REGIONNAME' - hbase> assign 'ENCODED_REGIONNAME' -EOF - end - - def command(region_name) - format_simple_command do - admin.assign(region_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb deleted file mode 100644 index 4d7778daa4..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balance_switch.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class BalanceSwitch < Command - def help - return <<-EOF -Enable/Disable balancer. Returns previous balancer state. -Examples: - - hbase> balance_switch true - hbase> balance_switch false -EOF - end - - def command(enableDisable) - format_simple_command do - formatter.row([ - admin.balance_switch(enableDisable)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb deleted file mode 100644 index c329eced4b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/balancer.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Balancer < Command - def help - return <<-EOF -Trigger the cluster balancer. Returns true if balancer ran and was able to -tell the region servers to unassign all the regions to balance (the re-assignment itself is async). -Otherwise false (Will not run if regions in transition). -EOF - end - - def command() - format_simple_command do - formatter.row([ - admin.balancer()? "true": "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb deleted file mode 100644 index b310c3a7ab..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CatalogjanitorEnabled < Command - def help - return <<-EOF -Query for the CatalogJanitor state (enabled/disabled?) -Examples: - - hbase> catalogjanitor_enabled -EOF - end - - def command() - format_simple_command do - formatter.row([ - admin.catalogjanitor_enabled()? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb deleted file mode 100644 index 03426cb4f1..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CatalogjanitorRun < Command - def help - return <<-EOF -Catalog janitor command to run the (garbage collection) scan from command line. - - hbase> catalogjanitor_run - -EOF - end - def command() - format_simple_command do - admin.catalogjanitor_run() - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb deleted file mode 100644 index fce1925eec..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CatalogjanitorSwitch < Command - def help - return <<-EOF -Enable/Disable CatalogJanitor. Returns previous CatalogJanitor state. -Examples: - - hbase> catalogjanitor_switch true - hbase> catalogjanitor_switch false -EOF - end - - def command(enableDisable) - format_simple_command do - formatter.row([ - admin.catalogjanitor_switch(enableDisable)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb deleted file mode 100644 index c4d5b0b2d5..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/clone_snapshot.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloneSnapshot < Command - def help - return <<-EOF -Create a new table by cloning the snapshot content. -There're no copies of data involved. -And writing on the newly created table will not influence the snapshot data. - -Examples: - hbase> clone_snapshot 'snapshotName', 'tableName' -EOF - end - - def command(snapshot_name, table) - format_simple_command do - admin.clone_snapshot(snapshot_name, table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb deleted file mode 100644 index b7cdf3dca2..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/close_region.rb +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CloseRegion < Command - def help - return <<-EOF -Close a single region. Ask the master to close a region out on the cluster -or if 'SERVER_NAME' is supplied, ask the designated hosting regionserver to -close the region directly. Closing a region, the master expects 'REGIONNAME' -to be a fully qualified region name. When asking the hosting regionserver to -directly close a region, you pass the regions' encoded name only. A region -name looks like this: - - TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. - -The trailing period is part of the regionserver name. A region's encoded name -is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 -(without the period). A 'SERVER_NAME' is its host, port plus startcode. For -example: host187.example.com,60020,1289493121758 (find servername in master ui -or when you do detailed status in shell). This command will end up running -close on the region hosting regionserver. The close is done without the -master's involvement (It will not know of the close). Once closed, region will -stay closed. Use assign to reopen/reassign. Use unassign or move to assign -the region elsewhere on cluster. Use with caution. For experts only. -Examples: - - hbase> close_region 'REGIONNAME' - hbase> close_region 'REGIONNAME', 'SERVER_NAME' - hbase> close_region 'ENCODED_REGIONNAME' - hbase> close_region 'ENCODED_REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(region_name, server = nil) - format_simple_command do - admin.close_region(region_name, server) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb deleted file mode 100644 index 47cf0380ac..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/compact.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Compact < Command - def help - return <<-EOF - Compact all regions in passed table or pass a region row - to compact an individual region. You can also compact a single column - family within a region. - Examples: - Compact all regions in a table: - hbase> compact 't1' - Compact an entire region: - hbase> compact 'r1' - Compact only a column family within a region: - hbase> compact 'r1', 'c1' - Compact a column family within a table: - hbase> compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb deleted file mode 100644 index e88c445d5b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/count.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Count < Command - def help - return <<-EOF -Count the number of rows in a table. Return value is the number of rows. -This operation may take a LONG time (Run '$HADOOP_HOME/bin/hadoop jar -hbase.jar rowcount' to run a counting mapreduce job). Current count is shown -every 1000 rows by default. Count interval may be optionally specified. Scan -caching is enabled on count scans by default. Default cache size is 10 rows. -If your rows are small in size, you may want to increase this -parameter. Examples: - - hbase> count 't1' - hbase> count 't1', INTERVAL => 100000 - hbase> count 't1', CACHE => 1000 - hbase> count 't1', INTERVAL => 10, CACHE => 1000 - -The same commands also can be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding commands would be: - - hbase> t.count - hbase> t.count INTERVAL => 100000 - hbase> t.count CACHE => 1000 - hbase> t.count INTERVAL => 10, CACHE => 1000 -EOF - end - - def command(table, params = {}) - count(table(table), params) - end - - def count(table, params = {}) - # If the second parameter is an integer, then it is the old command syntax - params = { 'INTERVAL' => params } if params.kind_of?(Fixnum) - - # Merge params with defaults - params = { - 'INTERVAL' => 1000, - 'CACHE' => 10 - }.merge(params) - - # Call the counter method - now = Time.now - formatter.header - count = table._count_internal(params['INTERVAL'].to_i, params['CACHE'].to_i) do |cnt, row| - formatter.row([ "Current count: #{cnt}, row: #{row}" ]) - end - formatter.footer(now, count) - return count - end - end - end -end - -#Add the method table.count that calls count.count -::Hbase::Table.add_shell_command("count") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb deleted file mode 100644 index e59a5cb65d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create.rb +++ /dev/null @@ -1,68 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Create < Command - def help - return <<-EOF -Creates a table. Pass a table name, and a set of column family -specifications (at least one), and, optionally, table configuration. -Column specification can be a simple string (name), or a dictionary -(dictionaries are described below in main help output), necessarily -including NAME attribute. -Examples: - - hbase> create 't1', {NAME => 'f1', VERSIONS => 5} - hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} - hbase> # The above in shorthand would be the following: - hbase> create 't1', 'f1', 'f2', 'f3' - hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} - hbase> create 't1', {NAME => 'f1', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} - -Table configuration options can be put at the end. -Examples: - - hbase> create 't1', 'f1', SPLITS => ['10', '20', '30', '40'] - hbase> create 't1', 'f1', SPLITS_FILE => 'splits.txt', OWNER => 'johndoe' - hbase> create 't1', {NAME => 'f1', VERSIONS => 5}, METADATA => { 'mykey' => 'myvalue' } - hbase> # Optionally pre-split the table into NUMREGIONS, using - hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) - hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} - hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'}} - -You can also keep around a reference to the created table: - - hbase> t1 = create 't1', 'f1' - -Which gives you a reference to the table named 't1', on which you can then -call methods. -EOF - end - - def command(table, *args) - format_simple_command do - ret = admin.create(table, *args) - end - #and then return the table you just created - table(table) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb deleted file mode 100644 index 3259eb6b11..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/create_namespace.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class CreateNamespace < Command - def help - return <<-EOF -Create namespace; pass namespace name, -and optionally a dictionary of namespace configuration. -Examples: - - hbase> create_namespace 'ns1' - hbase> create_namespace 'ns1', {'PROERTY_NAME'=>'PROPERTY_VALUE'} -EOF - end - - def command(namespace, *args) - format_simple_command do - admin.create_namespace(namespace, *args) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb deleted file mode 100644 index b0bca71277..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete.rb +++ /dev/null @@ -1,54 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Delete < Command - def help - return <<-EOF -Put a delete cell value at specified table/row/column and optionally -timestamp coordinates. Deletes must match the deleted cell's -coordinates exactly. When scanning, a delete cell suppresses older -versions. To delete a cell from 't1' at row 'r1' under column 'c1' -marked with the time 'ts1', do: - - hbase> delete 't1', 'r1', 'c1', ts1 - -The same command can also be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding command would be: - - hbase> t.delete 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - delete(table(table), row, column, timestamp) - end - - def delete(table, row, column, timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table._delete_internal(row, column, timestamp) - end - end - end - end -end - -#Add the method table.delete that calls delete.delete -::Hbase::Table.add_shell_command("delete") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb deleted file mode 100644 index b8c3791a54..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/delete_snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DeleteSnapshot < Command - def help - return <<-EOF -Delete a specified snapshot. Examples: - - hbase> delete_snapshot 'snapshotName', -EOF - end - - def command(snapshot_name) - format_simple_command do - admin.delete_snapshot(snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb deleted file mode 100644 index a09a882927..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/deleteall.rb +++ /dev/null @@ -1,57 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Deleteall < Command - def help - return <<-EOF -Delete all cells in a given row; pass a table name, row, and optionally -a column and timestamp. Examples: - - hbase> deleteall 't1', 'r1' - hbase> deleteall 't1', 'r1', 'c1' - hbase> deleteall 't1', 'r1', 'c1', ts1 - -The same commands also can be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding command would be: - - hbase> t.deleteall 'r1' - hbase> t.deleteall 'r1', 'c1' - hbase> t.deleteall 'r1', 'c1', ts1 -EOF - end - - def command(table, row, column = nil, - timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - deleteall(table(table), row, column, timestamp) - end - - def deleteall(table, row, column = nil, - timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP) - format_simple_command do - table._deleteall_internal(row, column, timestamp) - end - end - end - end -end - -#Add the method table.deleteall that calls deleteall.deleteall -::Hbase::Table.add_shell_command("deleteall") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb deleted file mode 100644 index cce82c4770..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Describe < Command - def help - return <<-EOF -Describe the named table. For example: - hbase> describe 't1' -EOF - end - - def command(table) - now = Time.now - - desc = admin.describe(table) - - formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) - formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb deleted file mode 100644 index cf135da558..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/describe_namespace.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DescribeNamespace < Command - def help - return <<-EOF -Describe the named namespace. For example: - hbase> describe_namespace 'ns1' -EOF - end - - def command(namespace) - now = Time.now - - desc = admin.describe_namespace(namespace) - - formatter.header([ "DESCRIPTION" ], [ 64 ]) - formatter.row([ desc ], true, [ 64 ]) - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb deleted file mode 100644 index b10aa9bf03..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Disable < Command - def help - return <<-EOF -Start disable of named table: e.g. "hbase> disable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.disable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb deleted file mode 100644 index 0e7c30ee99..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_all.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DisableAll < Command - def help - return <<-EOF -Disable all of tables matching the given regex: - -hbase> disable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.disable_all(regex) - puts "#{count - failed.size} tables successfully disabled" - puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb deleted file mode 100644 index 416545b21f..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/disable_peer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DisablePeer< Command - def help - return <<-EOF -Stops the replication stream to the specified cluster, but still -keeps track of new edits to replicate. - -Examples: - - hbase> disable_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.disable_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb deleted file mode 100644 index 6238fe65b8..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Drop < Command - def help - return <<-EOF -Drop the named table. Table must first be disabled: e.g. "hbase> drop 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.drop(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb deleted file mode 100644 index e482d6b0c7..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_all.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DropAll < Command - def help - return <<-EOF -Drop all of the tables matching the given regex: - -hbase> drop_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.drop_all(regex) - puts "#{count - failed.size} tables successfully dropped" - puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb deleted file mode 100644 index b030d279c7..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/drop_namespace.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class DropNamespace < Command - def help - return <<-EOF -Drop the named namespace. The namespace must be empty. -EOF - end - - def command(namespace) - format_simple_command do - admin.drop_namespace(namespace) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb deleted file mode 100644 index a4d495a55e..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Enable < Command - def help - return <<-EOF -Start enable of named table: e.g. "hbase> enable 't1'" -EOF - end - - def command(table) - format_simple_command do - admin.enable(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb deleted file mode 100644 index fc0f081f15..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_all.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class EnableAll < Command - def help - return <<-EOF -Enable all of the tables matching the given regex: - -hbase> enable_all 't.*' -EOF - end - - def command(regex) - regex = /^#{regex}$/ unless regex.is_a?(Regexp) - list = admin.list.grep(regex) - count = list.size - list.each do |table| - formatter.row([ table ]) - end - puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 - answer = 'n' - answer = gets.chomp unless count == 0 - puts "No tables matched the regex #{regex.to_s}" if count == 0 - return unless answer =~ /y.*/i - failed = admin.enable_all(regex) - puts "#{count - failed.size} tables successfully enabled" - puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.size == 0 - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb deleted file mode 100644 index 55136ffb7f..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/enable_peer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class EnablePeer< Command - def help - return <<-EOF -Restarts the replication to the specified peer cluster, -continuing from where it was disabled. - -Examples: - - hbase> enable_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.enable_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb deleted file mode 100644 index b7932ced52..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/exists.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Exists < Command - def help - return <<-EOF -Does the named table exist? e.g. "hbase> exists 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - "Table #{table} " + (admin.exists?(table.to_s) ? "does exist" : "does not exist") - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb deleted file mode 100644 index 2aefec52e9..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/flush.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Flush < Command - def help - return <<-EOF -Flush all regions in passed table or pass a region row to -flush an individual region. For example: - - hbase> flush 'TABLENAME' - hbase> flush 'REGIONNAME' - hbase> flush 'ENCODED_REGIONNAME' -EOF - end - - def command(table_or_region_name) - format_simple_command do - admin.flush(table_or_region_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb deleted file mode 100644 index 4344d68d14..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get.rb +++ /dev/null @@ -1,91 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Get < Command - def help - return <<-EOF -Get row or cell contents; pass table name, row, and optionally -a dictionary of column(s), timestamp, timerange and versions. Examples: - - hbase> get 't1', 'r1' - hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} - hbase> get 't1', 'r1', {COLUMN => 'c1'} - hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} - hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} - hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} - hbase> get 't1', 'r1', 'c1' - hbase> get 't1', 'r1', 'c1', 'c2' - hbase> get 't1', 'r1', ['c1', 'c2'] - -Besides the default 'toStringBinary' format, 'get' also supports custom formatting by -column. A user can define a FORMATTER by adding it to the column name in the get -specification. The FORMATTER can be stipulated: - - 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) - 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. - -Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: - hbase> get 't1', 'r1' {COLUMN => ['cf:qualifier1:toInt', - 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } - -Note that you can specify a FORMATTER by column only (cf:qualifer). You cannot specify -a FORMATTER for all columns of a column family. - -The same commands also can be run on a reference to a table (obtained via get_table or -create_table). Suppose you had a reference t to table 't1', the corresponding commands -would be: - - hbase> t.get 'r1' - hbase> t.get 'r1', {TIMERANGE => [ts1, ts2]} - hbase> t.get 'r1', {COLUMN => 'c1'} - hbase> t.get 'r1', {COLUMN => ['c1', 'c2', 'c3']} - hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} - hbase> t.get 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} - hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} - hbase> t.get 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} - hbase> t.get 'r1', 'c1' - hbase> t.get 'r1', 'c1', 'c2' - hbase> t.get 'r1', ['c1', 'c2'] -EOF - end - - def command(table, row, *args) - get(table(table), row, *args) - end - - def get(table, row, *args) - now = Time.now - formatter.header(["COLUMN", "CELL"]) - - table._get_internal(row, *args) do |column, value| - formatter.row([ column, value ]) - end - - formatter.footer(now) - end - end - end -end - -#add get command to table -::Hbase::Table.add_shell_command('get') diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb deleted file mode 100644 index ee71ae4978..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_counter.rb +++ /dev/null @@ -1,53 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class GetCounter < Command - def help - return <<-EOF -Return a counter cell value at specified table/row/column coordinates. -A cell cell should be managed with atomic increment function oh HBase -and the data should be binary encoded. Example: - - hbase> get_counter 't1', 'r1', 'c1' - -The same commands also can be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding command would be: - - hbase> t.get_counter 'r1', 'c1' -EOF - end - - def command(table, row, column, value) - get_counter(table(table), row, column, value) - end - - def get_counter(table, row, column, value = nil) - if cnt = table._get_counter_internal(row, column) - puts "COUNTER VALUE = #{cnt}" - else - puts "No counter found at specified coordinates" - end - end - end - end -end - -::Hbase::Table.add_shell_command('get_counter') diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb deleted file mode 100644 index 76f4b952d7..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/get_table.rb +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class GetTable < Command - def help - return <<-EOF -Get the given table name and return it as an actual object to -be manipulated by the user. See table.help for more information -on how to use the table. -Eg. - - hbase> t1 = get_table 't1' - -returns the table named 't1' as a table object. You can then do - - hbase> t1.help - -which will then print the help for that table. -EOF - end - - def command(table, *args) - format_and_return_simple_command do - table(table) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb deleted file mode 100644 index 7bdc95943b..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/grant.rb +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Grant < Command - def help - return <<-EOF -Grant users specific rights. -Syntax : grant [
[ []] - -permissions is either zero or more letters from the set "RWXCA". -READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') - -For example: - - hbase> grant 'bobsmith', 'RWXCA' - hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' -EOF - end - - def command(user, rights, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.grant(user, rights, table_name, family, qualifier) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb deleted file mode 100644 index b292791541..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/hlog_roll.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -module Shell - module Commands - class HlogRoll < Command - def help - return <<-EOF -Roll the log writer. That is, start writing log messages to a new file. -The name of the regionserver should be given as the parameter. A -'server_name' is the host, port plus startcode of a regionserver. For -example: host187.example.com,60020,1289493121758 (find servername in -master ui or when you do detailed status in shell) -EOF - end - - def command(server_name) - format_simple_command do - admin.hlog_roll(server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb deleted file mode 100644 index 6ebac29ace..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/incr.rb +++ /dev/null @@ -1,55 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Incr < Command - def help - return <<-EOF -Increments a cell 'value' at specified table/row/column coordinates. -To increment a cell value in table 't1' at row 'r1' under column -'c1' by 1 (can be omitted) or 10 do: - - hbase> incr 't1', 'r1', 'c1' - hbase> incr 't1', 'r1', 'c1', 1 - hbase> incr 't1', 'r1', 'c1', 10 - -The same commands also can be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding command would be: - - hbase> t.incr 'r1', 'c1' - hbase> t.incr 'r1', 'c1', 1 - hbase> t.incr 'r1', 'c1', 10 -EOF - end - - def command(table, row, column, value) - incr(table(table), row, column, value) - end - - def incr(table, row, column, value = nil) - cnt = table._incr_internal(row, column, value) - puts "COUNTER VALUE = #{cnt}" - end - end - end -end - -#add incr comamnd to Table -::Hbase::Table.add_shell_command("incr") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb deleted file mode 100644 index 28918ee429..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_disabled.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsDisabled < Command - def help - return <<-EOF -Is named table disabled?: e.g. "hbase> is_disabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.disabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb deleted file mode 100644 index 6a98a36b71..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/is_enabled.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class IsEnabled < Command - def help - return <<-EOF -Is named table enabled?: e.g. "hbase> is_enabled 't1'" -EOF - end - - def command(table) - format_simple_command do - formatter.row([ - admin.enabled?(table)? "true" : "false" - ]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb deleted file mode 100644 index 7801f530b9..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class List < Command - def help - return <<-EOF -List all tables in hbase. Optional regular expression parameter could -be used to filter the output. Examples: - - hbase> list - hbase> list 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "TABLE" ]) - - list = admin.list(regex) - list.each do |table| - formatter.row([ table ]) - end - - formatter.footer(now, list.size) - return list - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb deleted file mode 100644 index 0b577fe2bd..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListNamespace < Command - def help - return <<-EOF -List all namespaces in hbase. Optional regular expression parameter could -be used to filter the output. Examples: - - hbase> list_namespace - hbase> list_namespace 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "NAMESPACE" ]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_namespace.grep(regex) - list.each do |table| - formatter.row([ table ]) - end - - formatter.footer(now, list.size) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb deleted file mode 100644 index 29e18123e6..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_namespace_tables.rb +++ /dev/null @@ -1,45 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListNamespaceTables < Command - def help - return <<-EOF -List all tables that are members of the namespace. -Examples: - - hbase> list_namespace_tables 'ns1' -EOF - end - - def command(namespace) - now = Time.now - formatter.header([ "TABLE" ]) - - list = admin.list_namespace_tables(namespace) - list.each do |table| - formatter.row([ table ]) - end - - formatter.footer(now, list.size) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb deleted file mode 100644 index 2f57592e58..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_peers.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListPeers< Command - def help - return <<-EOF -List all replication peer clusters. - - hbase> list_peers -EOF - end - - def command() - now = Time.now - peers = replication_admin.list_peers - - formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE"]) - - peers.entrySet().each do |e| - state = replication_admin.get_peer_state(e.key) - formatter.row([ e.key, e.value, state ]) - end - - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb deleted file mode 100644 index b1494b8d3a..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_replicated_tables.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListReplicatedTables< Command - def help - return <<-EOF -List all the tables and column families replicated from this cluster - - hbase> list_replicated_tables - hbase> list_replicated_tables 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - - formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ]) - list = replication_admin.list_replicated_tables - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = list.select {|s| regex.match(s.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME))} - list.each do |e| - if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL - replicateType = "GLOBAL" - else - replicateType = "unknown" - end - formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32]) - end - formatter.footer(now) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb deleted file mode 100644 index 95135185c2..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/list_snapshots.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require 'time' - -module Shell - module Commands - class ListSnapshots < Command - def help - return <<-EOF -List all snapshots taken (by printing the names and relative information). -Optional regular expression parameter could be used to filter the output -by snapshot name. - -Examples: - hbase> list_snapshots - hbase> list_snapshots 'abc.*' -EOF - end - - def command(regex = ".*") - now = Time.now - formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) - - regex = /#{regex}/ unless regex.is_a?(Regexp) - list = admin.list_snapshot.select {|s| regex.match(s.getName)} - list.each do |snapshot| - creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s - formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) - end - - formatter.footer(now, list.size) - return list.map { |s| s.getName() } - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb deleted file mode 100644 index f05cc1d78d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/major_compact.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class MajorCompact < Command - def help - return <<-EOF - Run major compaction on passed table or pass a region row - to major compact an individual region. To compact a single - column family within a region specify the region name - followed by the column family name. - Examples: - Compact all regions in a table: - hbase> major_compact 't1' - Compact an entire region: - hbase> major_compact 'r1' - Compact a single column family within a region: - hbase> major_compact 'r1', 'c1' - Compact a single column family within a table: - hbase> major_compact 't1', 'c1' - EOF - end - - def command(table_or_region_name, family = nil) - format_simple_command do - admin.major_compact(table_or_region_name, family) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb deleted file mode 100644 index 6afa2e5b50..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/merge_region.rb +++ /dev/null @@ -1,49 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class MergeRegion < Command - def help - return <<-EOF -Merge two regions. Passing 'true' as the optional third parameter will force -a merge ('force' merges regardless else merge will fail unless passed -adjacent regions. 'force' is for expert use only). - -NOTE: You must pass the encoded region name, not the full region name so -this command is a little different from other region operations. The encoded -region name is the hash suffix on region names: e.g. if the region name were -TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then -the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 - -Examples: - - hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME' - hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', true -EOF - end - - def command(encoded_region_a_name, encoded_region_b_name, force = 'false') - format_simple_command do - admin.merge_region(encoded_region_a_name, encoded_region_b_name, force) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb deleted file mode 100644 index e6b28288e3..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/move.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Move < Command - def help - return <<-EOF -Move a region. Optionally specify target regionserver else we choose one -at random. NOTE: You pass the encoded region name, not the region name so -this command is a little different to the others. The encoded region name -is the hash suffix on region names: e.g. if the region name were -TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then -the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 -A server name is its host, port plus startcode. For example: -host187.example.com,60020,1289493121758 -Examples: - - hbase> move 'ENCODED_REGIONNAME' - hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' -EOF - end - - def command(encoded_region_name, server_name = nil) - format_simple_command do - admin.move(encoded_region_name, server_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb deleted file mode 100644 index 17e7a5f261..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/put.rb +++ /dev/null @@ -1,52 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Put < Command - def help - return <<-EOF -Put a cell 'value' at specified table/row/column and optionally -timestamp coordinates. To put a cell value into table 't1' at -row 'r1' under column 'c1' marked with the time 'ts1', do: - - hbase> put 't1', 'r1', 'c1', 'value', ts1 - -The same commands also can be run on a table reference. Suppose you had a reference -t to table 't1', the corresponding command would be: - - hbase> t.put 'r1', 'c1', 'value', ts1 -EOF - end - - def command(table, row, column, value, timestamp = nil) - put table(table), row, column, value, timestamp - end - - def put(table, row, column, value, timestamp = nil) - format_simple_command do - table._put_internal(row, column, value, timestamp) - end - end - end - end -end - -#Add the method table.put that calls Put.put -::Hbase::Table.add_shell_command("put") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb deleted file mode 100644 index 5ae5786fc4..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/remove_peer.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RemovePeer< Command - def help - return <<-EOF -Stops the specified replication stream and deletes all the meta -information kept about it. Examples: - - hbase> remove_peer '1' -EOF - end - - def command(id) - format_simple_command do - replication_admin.remove_peer(id) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb deleted file mode 100644 index ac9e23cbea..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/rename_snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RenameSnapshot < Command - def help - return <<-EOF -Rename a specified snapshot. Examples: - - hbase> rename_snapshot 'oldSnapshotName' 'newSnapshotName' -EOF - end - - def command(old_snapshot_name, new_snapshot_name) - format_simple_command do - admin.rename_snapshot(old_snapshot_name, new_snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb deleted file mode 100644 index 4d531711bc..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/restore_snapshot.rb +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class RestoreSnapshot < Command - def help - return <<-EOF -Restore a specified snapshot. -The restore will replace the content of the original table, -bringing back the content to the snapshot state. -The table must be disabled. - -Examples: - hbase> restore_snapshot 'snapshotName' -EOF - end - - def command(snapshot_name) - format_simple_command do - admin.restore_snapshot(snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb deleted file mode 100644 index 721519e7d6..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/revoke.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Revoke < Command - def help - return <<-EOF -Revoke a user's access rights. -Syntax : revoke [
[ []] -For example: - - hbase> revoke 'bobsmith' - hbase> revoke 'bobsmith', 't1', 'f1', 'col1' -EOF - end - - def command(user, table_name=nil, family=nil, qualifier=nil) - format_simple_command do - security_admin.revoke(user, table_name, family, qualifier) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb deleted file mode 100644 index 364fd6e0b1..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/scan.rb +++ /dev/null @@ -1,110 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Scan < Command - def help - return <<-EOF -Scan a table; pass table name and optionally a dictionary of scanner -specifications. Scanner specifications may include one or more of: -TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, -or COLUMNS, CACHE - -If no columns are specified, all columns will be scanned. -To scan all members of a column family, leave the qualifier empty as in -'col_family:'. - -The filter can be specified in two ways: -1. Using a filterString - more information on this is available in the -Filter Language document attached to the HBASE-4176 JIRA -2. Using the entire package name of the filter. - -Some examples: - - hbase> scan 'hbase:meta' - hbase> scan 'hbase:meta', {COLUMNS => 'info:regioninfo'} - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} - hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} - hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND - (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} - hbase> scan 't1', {FILTER => - org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} - -For experts, there is an additional option -- CACHE_BLOCKS -- which -switches block caching for the scanner on (true) or off (false). By -default it is enabled. Examples: - - hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} - -Also for experts, there is an advanced option -- RAW -- which instructs the -scanner to return all cells (including delete markers and uncollected deleted -cells). This option cannot be combined with requesting specific COLUMNS. -Disabled by default. Example: - - hbase> scan 't1', {RAW => true, VERSIONS => 10} - -Besides the default 'toStringBinary' format, 'scan' supports custom formatting -by column. A user can define a FORMATTER by adding it to the column name in -the scan specification. The FORMATTER can be stipulated: - - 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) - 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. - -Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: - hbase> scan 't1', {COLUMNS => ['cf:qualifier1:toInt', - 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } - -Note that you can specify a FORMATTER by column only (cf:qualifer). You cannot -specify a FORMATTER for all columns of a column family. - -Scan can also be used directly from a table, by first getting a reference to a -table, like such: - - hbase> t = get_table 't' - hbase> t.scan - -Note in the above situation, you can still provide all the filtering, columns, -options, etc as described above. - -EOF - end - - def command(table, args = {}) - scan(table(table), args) - end - - #internal command that actually does the scanning - def scan(table, args = {}) - now = Time.now - formatter.header(["ROW", "COLUMN+CELL"]) - - #actually do the scanning - count = table._scan_internal(args) do |row, cells| - formatter.row([ row, cells ]) - end - - formatter.footer(now, count) - end - end - end -end - -#Add the method table.scan that calls Scan.scan -::Hbase::Table.add_shell_command("scan") diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb deleted file mode 100644 index 716859fc06..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/show_filters.rb +++ /dev/null @@ -1,54 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -java_import org.apache.hadoop.hbase.filter.ParseFilter - -module Shell - module Commands - class ShowFilters < Command - def help - return <<-EOF -Show all the filters in hbase. Example: - hbase> show_filters - - Documentation on filters mentioned below can be found at: https://our.intern.facebook.com/intern/wiki/index.php/HBase/Filter_Language - ColumnPrefixFilter - TimestampsFilter - PageFilter - ..... - KeyOnlyFilter -EOF - end - - def command( ) - now = Time.now - formatter.row(["Documentation on filters mentioned below can " + - "be found at: https://our.intern.facebook.com/intern/" + - "wiki/index.php/HBase/Filter_Language"]) - - parseFilter = ParseFilter.new - supportedFilters = parseFilter.getSupportedFilters - - supportedFilters.each do |filter| - formatter.row([filter]) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb deleted file mode 100644 index 1c4ecfec0e..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/snapshot.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Snapshot < Command - def help - return <<-EOF -Take a snapshot of specified table. Examples: - - hbase> snapshot 'sourceTable', 'snapshotName' -EOF - end - - def command(table, snapshot_name) - format_simple_command do - admin.snapshot(table, snapshot_name) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb deleted file mode 100644 index 9488ed9ef1..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/split.rb +++ /dev/null @@ -1,42 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Split < Command - def help - return <<-EOF -Split entire table or pass a region to split individual region. With the -second parameter, you can specify an explicit split key for the region. -Examples: - split 'tableName' - split 'regionName' # format: 'tableName,startKey,id' - split 'tableName', 'splitKey' - split 'regionName', 'splitKey' -EOF - end - - def command(table_or_region_name, split_point = nil) - format_simple_command do - admin.split(table_or_region_name, split_point) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb deleted file mode 100644 index f72c13caef..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/status.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Status < Command - def help - return <<-EOF -Show cluster status. Can be 'summary', 'simple', or 'detailed'. The -default is 'summary'. Examples: - - hbase> status - hbase> status 'simple' - hbase> status 'summary' - hbase> status 'detailed' -EOF - end - - def command(format = 'summary') - admin.status(format) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb deleted file mode 100644 index e5a1858a92..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/table_help.rb +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class TableHelp < Command - def help - return Hbase::Table.help - end - - #just print the help - def command - # call the shell to get the nice formatting there - @shell.help_command 'table_help' - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb deleted file mode 100644 index ba061957c2..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/trace.rb +++ /dev/null @@ -1,76 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -HTrace = org.cloudera.htrace.Trace -java_import org.cloudera.htrace.Sampler -java_import org.apache.hadoop.hbase.trace.SpanReceiverHost - -module Shell - module Commands - class Trace < Command - def help - return <<-EOF -Start or Stop tracing using HTrace. -Always returns true if tracing is running, otherwise false. -If the first argument is 'start', new span is started. -If the first argument is 'stop', current running span is stopped. -('stop' returns false on success.) -If the first argument is 'status', just returns if or not tracing is running. -On 'start'-ing, you can optionally pass the name of span as the second argument. -The default name of span is 'HBaseShell'. -Repeating 'start' does not start nested span. - -Examples: - - hbase> trace 'start' - hbase> trace 'status' - hbase> trace 'stop' - - hbase> trace 'start', 'MySpanName' - hbase> trace 'stop' - -EOF - end - - def command(startstop="status", spanname="HBaseShell") - format_and_return_simple_command do - trace(startstop, spanname) - end - end - - def trace(startstop, spanname) - @@receiver ||= SpanReceiverHost.getInstance(@shell.hbase.configuration) - if startstop == "start" - if not tracing? - @@tracescope = HTrace.startSpan(spanname, Sampler.ALWAYS) - end - elsif startstop == "stop" - if tracing? - @@tracescope.close() - end - end - tracing? - end - - def tracing?() - HTrace.isTracing() - end - - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb deleted file mode 100644 index b7812fbe3d..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Truncate < Command - def help - return <<-EOF - Disables, drops and recreates the specified table. -EOF - end - - def command(table) - format_simple_command do - puts "Truncating '#{table}' table (it may take a while):" - admin.truncate(table) { |log| puts " - #{log}" } - end - end - - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb deleted file mode 100644 index 918b23289a..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/truncate_preserve.rb +++ /dev/null @@ -1,38 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class TruncatePreserve < Command - def help - return <<-EOF - Disables, drops and recreates the specified table while still maintaing the previous region boundaries. -EOF - end - - def command(table) - format_simple_command do - puts "Truncating '#{table}' table (it may take a while):" - admin.truncate_preserve(table) { |log| puts " - #{log}" } - end - end - - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb deleted file mode 100644 index 5eea71f6e0..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/unassign.rb +++ /dev/null @@ -1,45 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Unassign < Command - def help - return <<-EOF -Unassign a region. Unassign will close region in current location and then -reopen it again. Pass 'true' to force the unassignment ('force' will clear -all in-memory state in master before the reassign. If results in -double assignment use hbck -fix to resolve. To be used by experts). -Use with caution. For expert use only. Examples: - - hbase> unassign 'REGIONNAME' - hbase> unassign 'REGIONNAME', true - hbase> unassign 'ENCODED_REGIONNAME' - hbase> unassign 'ENCODED_REGIONNAME', true -EOF - end - - def command(region_name, force = 'false') - format_simple_command do - admin.unassign(region_name, force) - end - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb deleted file mode 100644 index ad4a7b029e..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/user_permission.rb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class UserPermission < Command - def help - return <<-EOF -Show all permissions for the particular user. -Syntax : user_permission
-For example: - - hbase> user_permission - hbase> user_permission 'table1' -EOF - end - - def command(table=nil) - #format_simple_command do - #admin.user_permission(table) - now = Time.now - formatter.header(["User", "Table,Family,Qualifier:Permission"]) - - count = security_admin.user_permission(table) do |user, permission| - formatter.row([ user, permission]) - end - - formatter.footer(now, count) - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb deleted file mode 100644 index 63e9712a52..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/version.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Version < Command - def help - return <<-EOF -Output this HBase version -EOF - end - - def command - # Output version. - puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + - "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + - "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb deleted file mode 100644 index 040ad7e7ef..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/whoami.rb +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class Whoami < Command - def help - return <<-EOF -Show the current hbase user. -Syntax : whoami -For example: - - hbase> whoami -EOF - end - - def command() - puts "#{org.apache.hadoop.hbase.security.User.getCurrent().toString()}" - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb deleted file mode 100644 index c0b509a3c6..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/commands/zk_dump.rb +++ /dev/null @@ -1,34 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ZkDump < Command - def help - return <<-EOF -Dump status of HBase cluster as seen by ZooKeeper. -EOF - end - - def command - puts admin.zk_dump - end - end - end -end diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb b/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb deleted file mode 100644 index 36aaf76aea..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/ruby/shell/formatter.rb +++ /dev/null @@ -1,170 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Results formatter -module Shell - module Formatter - # Base abstract class for results formatting. - class Base - attr_reader :row_count - - def is_valid_io?(obj) - obj.instance_of?(IO) || obj == Kernel - end - - def refresh_width() - if $stdout.tty? - @max_width = Java::jline.Terminal.getTerminal().getTerminalWidth() - else - @max_width = 0 - end - end - - # Takes an output stream and a print width. - def initialize(opts = {}) - options = { - :output_stream => Kernel, - }.merge(opts) - - @out = options[:output_stream] - refresh_width - @row_count = 0 - - # raise an error if the stream is not valid - raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) - end - - def header(args = [], widths = []) - refresh_width - row(args, false, widths) if args.length > 0 - @row_count = 0 - end - - # Output a row. - # Inset is whether or not to offset row by a space. - def row(args = [], inset = true, widths = []) - # Print out nothing - return if !args || args.empty? - - # Print a string - if args.is_a?(String) - output(args) - @out.puts - return - end - - # TODO: Look at the type. Is it RowResult? - if args.length == 1 - splits = split(@max_width, dump(args[0])) - for l in splits - output(@max_width, l) - @out.puts - end - elsif args.length == 2 - if @max_width == 0 - col1width = col2width = 0 - else - col1width = (not widths or widths.length == 0) ? @max_width / 4 : @max_width * widths[0] / 100 - col2width = (not widths or widths.length < 2) ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 - end - splits1 = split(col1width, dump(args[0])) - splits2 = split(col2width, dump(args[1])) - biggest = (splits2.length > splits1.length)? splits2.length: splits1.length - index = 0 - while index < biggest - # Inset by one space if inset is set. - @out.print(" ") if inset - output(col1width, splits1[index]) - # Add extra space so second column lines up w/ second column output - @out.print(" ") unless inset - @out.print(" ") - output(col2width, splits2[index]) - index += 1 - @out.puts - end - else - # Print a space to set off multi-column rows - print ' ' - first = true - for e in args - @out.print " " unless first - first = false - @out.print e - end - puts - end - @row_count += 1 - end - - def split(width, str) - if width == 0 - return [str] - end - result = [] - index = 0 - while index < str.length do - result << str.slice(index, width) - index += width - end - result - end - - def dump(str) - return if str.instance_of?(Fixnum) - # Remove double-quotes added by 'dump'. - return str - end - - def output(str) - output(@max_width, str) - end - - def output(width, str) - if str == nil - str = '' - end - if not width or width == str.length - @out.print(str) - else - @out.printf('%-*s', width, str) - end - end - - def footer(start_time = nil, row_count = nil) - return unless start_time - row_count ||= @row_count - # Only output elapsed time and row count if startTime passed - @out.puts("%d row(s) in %.4f seconds" % [row_count, Time.now - start_time]) - end - end - - - class Console < Base - end - - class XHTMLFormatter < Base - # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html - # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html - end - - class JSON < Base - end - end -end - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh deleted file mode 100755 index 3b096f01a3..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/start-hbase.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -# Start hadoop hbase daemons. -# Run this on master node. -usage="Usage: start-hbase.sh" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - - -if [ "$1" = "autorestart" ] -then - commandToRun="autorestart" -else - commandToRun="start" -fi - -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` - - -if [ "$distMode" == 'false' ] -then - "$bin"/hbase-daemon.sh $commandToRun master $@ -else - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" $commandToRun zookeeper - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" $commandToRun regionserver - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" $commandToRun master-backup -fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh deleted file mode 100755 index a9ad3460c6..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/stop-hbase.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -# Stop hadoop hbase daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh -. "$bin"/hbase-common.sh - -# variables needed for stop command -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid - -echo -n stopping hbase -echo "`date` Stopping hbase (via master)" >> $loglog - -nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - master stop "$@" > "$logout" 2>&1 < /dev/null & - -waitForProcessEnd `cat $pid` 'stop-master-command' - -rm -f $pid - - -# distributed == false means that the HMaster will kill ZK when it exits -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'true' ] -then - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper -fi diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh deleted file mode 100755 index eb8633f502..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/test/process_based_cluster.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# -#/** -# * Copyright The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e -u -o pipefail - -SCRIPT_NAME=${0##*/} -SCRIPT_DIR=$(cd `dirname $0` && pwd ) - -print_usage() { - cat >&2 < -Options: - --kill - Kill local process-based HBase cluster using pid files. - --show - Show HBase processes running on this machine -EOT - exit 1 -} - -show_processes() { - ps -ef | grep -P "(HRegionServer|HMaster|HQuorumPeer) start" | grep -v grep -} - -cmd_specified() { - if [ "$CMD_SPECIFIED" ]; then - echo "Only one command can be specified" >&2 - exit 1 - fi - CMD_SPECIFIED=1 -} - -list_pid_files() { - LOCAL_CLUSTER_DIR=$SCRIPT_DIR/../../target/local_cluster - LOCAL_CLUSTER_DIR=$( cd $LOCAL_CLUSTER_DIR && pwd ) - find $LOCAL_CLUSTER_DIR -name "*.pid" -} - -if [ $# -eq 0 ]; then - print_usage -fi - -IS_KILL="" -IS_SHOW="" -CMD_SPECIFIED="" - -while [ $# -ne 0 ]; do - case "$1" in - -h|--help) - print_usage ;; - --kill) - IS_KILL=1 - cmd_specified ;; - --show) - IS_SHOW=1 - cmd_specified ;; - *) - echo "Invalid option: $1" >&2 - exit 1 - esac - shift -done - -if [ "$IS_KILL" ]; then - list_pid_files | \ - while read F; do - PID=`cat $F` - echo "Killing pid $PID from file $F" - # Kill may fail but that's OK, so turn off error handling for a moment. - set +e - kill -9 $PID - set -e - done -elif [ "$IS_SHOW" ]; then - PIDS="" - for F in `list_pid_files`; do - PID=`cat $F` - if [ -n "$PID" ]; then - if [ -n "$PIDS" ]; then - PIDS="$PIDS," - fi - PIDS="$PIDS$PID" - fi - done - ps -p $PIDS -else - echo "No command specified" >&2 - exit 1 -fi - - diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch b/titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch deleted file mode 100644 index e17adde94e..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/titan_customizations_to_hbase_script.patch +++ /dev/null @@ -1,46 +0,0 @@ ---- /home/dalaro/hbase-0.96.1.1-hadoop1/bin/hbase 2013-12-17 14:05:37.000000000 -0500 -+++ /home/dalaro/tinkerelius/titan/titan-hbase/titan-hbase-096/src/test/bin/hbase 2014-02-12 11:25:18.000000000 -0500 -@@ -143,10 +143,24 @@ - - add_maven_deps_to_classpath() { - f="${HBASE_HOME}/target/cached_classpath.txt" -- if [ ! -f "${f}" ] -- then -- echo "As this is a development environment, we need ${f} to be generated from maven (command: mvn install -DskipTests)" -- exit 1 -+ if [ ! -f "${f}" ]; then -+ # ------------------- -+ # Titan build support -+ # ------------------- -+ # Check for Maven command line options file. -+ # If titan-hbase has already been built, then this file will contain options -+ # to target the version of Hadoop used during compilation. This is -+ # necessary when the user compiles titan-hbase with a non-default Hadoop -+ # version (e.g. 2.2.0). If we omitted this check, then we'd try to start -+ # HBase with the default Hadoop version (currentl 1.0.4), and it would break. -+ MVN_OPTS= -+ optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" -+ unset IFS -+ if [ -f "${optf}" ]; then -+ MVN_OPTS=`cat "${optf}"` -+ fi -+ mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null -+ IFS= - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` - } -@@ -262,12 +276,8 @@ - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi -- #find the hbase ruby sources -- if [ -d "$HBASE_HOME/lib/ruby" ]; then -- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/lib/ruby" -- else -- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/hbase-shell/src/main/ruby" -- fi -+ # find the hbase ruby sources -+ HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" - elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' diff --git a/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh b/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh deleted file mode 100755 index 89a214e5a8..0000000000 --- a/titan-hbase-parent/titan-hbase-096/src/test/bin/zookeepers.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -if [ "$HBASE_MANAGES_ZK" = "" ]; then - HBASE_MANAGES_ZK=true -fi - -if [ "$HBASE_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait From 11a8cb2884f0be331a224f81d518e61479885f7f Mon Sep 17 00:00:00 2001 From: David Robinson Date: Mon, 14 Dec 2015 09:54:28 -0500 Subject: [PATCH 2/3] fix hbase 0.98 version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4d19560aaf..0003f8a6e3 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 4.4.1 1.2.1 2.7.1 - 0.98.16.1 + 0.98.2 ${hbase098.core.version}-hadoop2 1.0.2 ${hbase100.core.version} From c208ea441bc5b88fa67cc80e07aa0bdb7414d901 Mon Sep 17 00:00:00 2001 From: David Robinson Date: Mon, 14 Dec 2015 14:15:27 -0500 Subject: [PATCH 3/3] more changes that depend on hbase/hadoop changes --- pom.xml | 2 +- titan-all/pom.xml | 11 +- titan-dist/pom.xml | 5 +- titan-dist/titan-dist-hadoop-1/pom.xml | 214 ---- .../src/assembly/descriptor/archive.xml | 18 - titan-dist/titan-dist-hadoop-2/pom.xml | 14 +- titan-hadoop-parent/pom.xml | 1 - titan-hadoop-parent/titan-hadoop-1/pom.xml | 112 -- .../hadoop/compat/h1/DistCacheConfigurer.java | 36 - .../titan/hadoop/compat/h1/Hadoop1Compat.java | 89 -- .../compat/h1/Hadoop1CountersScanMetrics.java | 39 - .../compat/h1/ImmutableConfiguration.java | 283 ----- .../hadoop/compat/h1/MapredJarConfigurer.java | 20 - .../formats/TitanH1OutputCommitter.java | 40 - .../hadoop/formats/TitanH1OutputFormat.java | 103 -- .../hadoop/formats/TitanH1RecordWriter.java | 53 - titan-hadoop-parent/titan-hadoop/pom.xml | 6 - titan-hbase-parent/pom.xml | 1 + titan-hbase-parent/titan-hbase-11/pom.xml | 167 +++ titan-hbase-parent/titan-hbase-11/readme.txt | 1 + .../diskstorage/hbase/HBaseAdmin1_1.java | 127 +++ .../diskstorage/hbase/HBaseCompat1_1.java | 45 + .../diskstorage/hbase/HConnection1_1.java | 36 + .../titan/diskstorage/hbase/HTable1_1.java | 47 + .../src/test/bin/draining_servers.rb | 164 +++ .../src/test/bin/get-active-master.rb | 44 + .../src/test/bin/graceful_stop.sh | 165 +++ .../titan-hbase-11/src/test/bin/hbase | 421 ++++++++ .../src/test/bin/hbase-cleanup.sh | 147 +++ .../src/test/bin/hbase-common.sh | 41 + .../src/test/bin/hbase-config.cmd | 78 ++ .../src/test/bin/hbase-config.sh | 136 +++ .../src/test/bin/hbase-daemon.sh | 284 +++++ .../src/test/bin/hbase-daemons.sh | 53 + .../titan-hbase-11/src/test/bin/hbase-jruby | 22 + .../titan-hbase-11/src/test/bin/hbase.cmd | 439 ++++++++ .../titan-hbase-11/src/test/bin/hirb.rb | 195 ++++ .../src/test/bin/local-master-backup.sh | 60 ++ .../src/test/bin/local-regionservers.sh | 58 ++ .../src/test/bin/master-backup.sh | 74 ++ .../src/test/bin/region_mover.rb | 488 +++++++++ .../src/test/bin/region_status.rb | 153 +++ .../src/test/bin/regionservers.sh | 78 ++ .../test/bin/replication/copy_tables_desc.rb | 73 ++ .../src/test/bin/rolling-restart.sh | 174 ++++ .../titan-hbase-11/src/test/bin/ruby/hbase.rb | 94 ++ .../src/test/bin/ruby/hbase/admin.rb | 973 ++++++++++++++++++ .../src/test/bin/ruby/hbase/hbase.rb | 70 ++ .../test/bin/ruby/hbase/replication_admin.rb | 174 ++++ .../src/test/bin/ruby/hbase/security.rb | 190 ++++ .../src/test/bin/ruby/hbase/table.rb | 687 +++++++++++++ .../test/bin/ruby/hbase/visibility_labels.rb | 148 +++ .../src/test/bin/ruby/irb/hirb.rb | 59 ++ .../titan-hbase-11/src/test/bin/ruby/shell.rb | 395 +++++++ .../src/test/bin/ruby/shell/commands.rb | 138 +++ .../bin/ruby/shell/commands/add_labels.rb | 40 + .../test/bin/ruby/shell/commands/add_peer.rb | 70 ++ .../src/test/bin/ruby/shell/commands/alter.rb | 101 ++ .../bin/ruby/shell/commands/alter_async.rb | 65 ++ .../ruby/shell/commands/alter_namespace.rb | 44 + .../bin/ruby/shell/commands/alter_status.rb | 38 + .../test/bin/ruby/shell/commands/append.rb | 52 + .../shell/commands/append_peer_tableCFs.rb | 41 + .../test/bin/ruby/shell/commands/assign.rb | 41 + .../bin/ruby/shell/commands/balance_switch.rb | 42 + .../test/bin/ruby/shell/commands/balancer.rb | 40 + .../shell/commands/catalogjanitor_enabled.rb | 40 + .../ruby/shell/commands/catalogjanitor_run.rb | 37 + .../shell/commands/catalogjanitor_switch.rb | 41 + .../bin/ruby/shell/commands/clear_auths.rb | 40 + .../bin/ruby/shell/commands/clone_snapshot.rb | 41 + .../bin/ruby/shell/commands/close_region.rb | 61 ++ .../test/bin/ruby/shell/commands/compact.rb | 48 + .../bin/ruby/shell/commands/compact_rs.rb | 43 + .../src/test/bin/ruby/shell/commands/count.rb | 77 ++ .../test/bin/ruby/shell/commands/create.rb | 72 ++ .../ruby/shell/commands/create_namespace.rb | 41 + .../test/bin/ruby/shell/commands/delete.rb | 59 ++ .../shell/commands/delete_all_snapshot.rb | 61 ++ .../ruby/shell/commands/delete_snapshot.rb | 37 + .../test/bin/ruby/shell/commands/deleteall.rb | 60 ++ .../test/bin/ruby/shell/commands/describe.rb | 50 + .../ruby/shell/commands/describe_namespace.rb | 41 + .../test/bin/ruby/shell/commands/disable.rb | 38 + .../bin/ruby/shell/commands/disable_all.rb | 50 + .../bin/ruby/shell/commands/disable_peer.rb | 41 + .../commands/disable_table_replication.rb | 40 + .../src/test/bin/ruby/shell/commands/drop.rb | 38 + .../test/bin/ruby/shell/commands/drop_all.rb | 50 + .../bin/ruby/shell/commands/drop_namespace.rb | 36 + .../test/bin/ruby/shell/commands/enable.rb | 38 + .../bin/ruby/shell/commands/enable_all.rb | 50 + .../bin/ruby/shell/commands/enable_peer.rb | 41 + .../commands/enable_table_replication.rb | 40 + .../test/bin/ruby/shell/commands/exists.rb | 40 + .../src/test/bin/ruby/shell/commands/flush.rb | 41 + .../src/test/bin/ruby/shell/commands/get.rb | 98 ++ .../test/bin/ruby/shell/commands/get_auths.rb | 43 + .../bin/ruby/shell/commands/get_counter.rb | 54 + .../test/bin/ruby/shell/commands/get_table.rb | 47 + .../src/test/bin/ruby/shell/commands/grant.rb | 118 +++ .../src/test/bin/ruby/shell/commands/incr.rb | 65 ++ .../bin/ruby/shell/commands/is_disabled.rb | 40 + .../bin/ruby/shell/commands/is_enabled.rb | 40 + .../src/test/bin/ruby/shell/commands/list.rb | 49 + .../bin/ruby/shell/commands/list_labels.rb | 44 + .../bin/ruby/shell/commands/list_namespace.rb | 46 + .../shell/commands/list_namespace_tables.rb | 45 + .../bin/ruby/shell/commands/list_peers.rb | 48 + .../shell/commands/list_replicated_tables.rb | 50 + .../bin/ruby/shell/commands/list_snapshots.rb | 51 + .../bin/ruby/shell/commands/major_compact.rb | 49 + .../bin/ruby/shell/commands/merge_region.rb | 49 + .../src/test/bin/ruby/shell/commands/move.rb | 47 + .../src/test/bin/ruby/shell/commands/put.rb | 57 + .../bin/ruby/shell/commands/remove_peer.rb | 39 + .../shell/commands/remove_peer_tableCFs.rb | 42 + .../ruby/shell/commands/restore_snapshot.rb | 41 + .../test/bin/ruby/shell/commands/revoke.rb | 48 + .../src/test/bin/ruby/shell/commands/scan.rb | 115 +++ .../test/bin/ruby/shell/commands/set_auths.rb | 40 + .../ruby/shell/commands/set_peer_tableCFs.rb | 47 + .../bin/ruby/shell/commands/set_visibility.rb | 73 ++ .../bin/ruby/shell/commands/show_filters.rb | 49 + .../ruby/shell/commands/show_peer_tableCFs.rb | 37 + .../test/bin/ruby/shell/commands/snapshot.rb | 38 + .../src/test/bin/ruby/shell/commands/split.rb | 43 + .../test/bin/ruby/shell/commands/status.rb | 43 + .../bin/ruby/shell/commands/table_help.rb | 33 + .../src/test/bin/ruby/shell/commands/trace.rb | 76 ++ .../test/bin/ruby/shell/commands/truncate.rb | 38 + .../ruby/shell/commands/truncate_preserve.rb | 38 + .../test/bin/ruby/shell/commands/unassign.rb | 45 + .../ruby/shell/commands/update_all_config.rb | 39 + .../bin/ruby/shell/commands/update_config.rb | 40 + .../ruby/shell/commands/user_permission.rb | 54 + .../test/bin/ruby/shell/commands/version.rb | 37 + .../test/bin/ruby/shell/commands/wal_roll.rb | 42 + .../test/bin/ruby/shell/commands/whoami.rb | 42 + .../test/bin/ruby/shell/commands/zk_dump.rb | 34 + .../src/test/bin/ruby/shell/formatter.rb | 170 +++ .../src/test/bin/shutdown_regionserver.rb | 52 + .../src/test/bin/start-hbase.cmd | 61 ++ .../src/test/bin/start-hbase.sh | 61 ++ .../src/test/bin/stop-hbase.cmd | 54 + .../titan-hbase-11/src/test/bin/stop-hbase.sh | 65 ++ .../test/bin/test/process_based_cluster.sh | 110 ++ .../src/test/bin/thread-pool.rb | 50 + ...titan_customizations_to_hbase_script.patch | 50 + .../titan-hbase-11/src/test/bin/zookeepers.sh | 59 ++ 150 files changed, 11631 insertions(+), 1025 deletions(-) delete mode 100644 titan-dist/titan-dist-hadoop-1/pom.xml delete mode 100644 titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml delete mode 100644 titan-hadoop-parent/titan-hadoop-1/pom.xml delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java delete mode 100644 titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java create mode 100644 titan-hbase-parent/titan-hbase-11/pom.xml create mode 100644 titan-hbase-parent/titan-hbase-11/readme.txt create mode 100644 titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java create mode 100644 titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java create mode 100644 titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java create mode 100644 titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/local-master-backup.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb create mode 100644 titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch create mode 100755 titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh diff --git a/pom.xml b/pom.xml index 0003f8a6e3..b4acbb30f7 100644 --- a/pom.xml +++ b/pom.xml @@ -69,12 +69,12 @@ 2.7.10 1.7.5 4.4.1 - 1.2.1 2.7.1 0.98.2 ${hbase098.core.version}-hadoop2 1.0.2 ${hbase100.core.version} + 1.1.1 1.9.2 2.3.0 - + titan-dist-hadoop-2 + ${project.groupId} diff --git a/titan-dist/titan-dist-hadoop-1/pom.xml b/titan-dist/titan-dist-hadoop-1/pom.xml deleted file mode 100644 index b91030b798..0000000000 --- a/titan-dist/titan-dist-hadoop-1/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-dist - 1.0.1-SNAPSHOT - ../pom.xml - - pom - titan-dist-hadoop-1 - Titan-Dist: Archive with Hadoop 1 - http://thinkaurelius.github.com/titan/ - - - hadoop1 - ${project.parent.basedir}/src/assembly/descriptor - ${project.parent.basedir}/src/assembly/static - ${project.parent.basedir}/src/assembly/resources - ${project.parent.build.directory}/cfilter - ${project.parent.parent.basedir} - - - - - ${project.groupId} - titan-all - ${project.version} - pom - - - org.apache.hadoop - hadoop-client - - - org.apache.hbase - hbase-client - - - - ${project.groupId} - titan-solr - - - - - org.apache.hbase - hbase-client - ${hbase098.core.version}-hadoop1 - - - - org.jruby - jruby-complete - - - - - org.apache.hadoop - hadoop-client - ${hadoop1.version} - - - - - - - maven-assembly-plugin - - - - org.codehaus.mojo - exec-maven-plugin - - - generate-titan-gremlin-imports - generate-resources - - - - - - org.codehaus.mojo - properties-maven-plugin - - - - maven-gpg-plugin - - - - - - - aurelius-release - - - - maven-failsafe-plugin - - - - org.codehaus.mojo - wagon-maven-plugin - - - - maven-resources-plugin - - - filter-cassandra-murmur-config - process-test-resources - - - filter-cassandra-bop-config - process-test-resources - - - filter-expect-scripts - process-test-resources - - - filter-test-resources - process-test-resources - - - copy-test-cfiles - process-test-resources - - - - - - - - - - - - dev-install-hadoop1 - - - - dev.hadoop - 1 - - - - - - - maven-clean-plugin - - - clean-dev-dirs-hadoop - clean - - - - - - maven-assembly-plugin - - - install-dev-dirs-hadoop - install - - - - - - - - - dev-install-hadoop1-by-default - - - - - !dev.hadoop - - - - - - - maven-clean-plugin - - - clean-dev-dirs-hadoop - clean - - - - - - maven-assembly-plugin - - - install-dev-dirs-hadoop - install - - - - - - - - diff --git a/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml b/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml deleted file mode 100644 index 399f49ea64..0000000000 --- a/titan-dist/titan-dist-hadoop-1/src/assembly/descriptor/archive.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - titan-${project.version}-${hadoop.version.tag} - titan-${project.version}-${hadoop.version.tag} - - - ${assembly.descriptor.dir}/archive.xml - - - - - src/assembly/static - / - false - - - diff --git a/titan-dist/titan-dist-hadoop-2/pom.xml b/titan-dist/titan-dist-hadoop-2/pom.xml index fa5fdaa041..88841f8398 100644 --- a/titan-dist/titan-dist-hadoop-2/pom.xml +++ b/titan-dist/titan-dist-hadoop-2/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.9.0-SNAPSHOT + 1.0.1-SNAPSHOT ../pom.xml pom @@ -24,13 +24,19 @@ org.apache.hbase hbase-client - ${hbase098.version} + ${hbase111.version} org.apache.hbase hbase-server + ${hbase111.version} - ${hbase098.version} + + + com.lmax + disruptor + + @@ -151,7 +157,7 @@ -classpath com.thinkaurelius.titan.example.GraphOfTheGodsFactory - ${project.build.directory}/conf/titan-berkeleydb-es.properties + ${project.build.directory}/conf/titan-berkeleyje-es.properties diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index 363c4f0c50..3456d7bb2e 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -20,7 +20,6 @@ titan-hadoop-core - titan-hadoop-1 titan-hadoop-2 titan-hadoop diff --git a/titan-hadoop-parent/titan-hadoop-1/pom.xml b/titan-hadoop-parent/titan-hadoop-1/pom.xml deleted file mode 100644 index 69cb61d6b9..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/pom.xml +++ /dev/null @@ -1,112 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan-hadoop-parent - 1.0.1-SNAPSHOT - ../pom.xml - - titan-hadoop-1 - Titan-Hadoop: 1.x Compatibility Shim - http://thinkaurelius.github.com/titan/ - - - ${basedir}/../.. - - - - - org.apache.hadoop - hadoop-core - ${hadoop1.version} - - - javax.servlet - jsp-api - - - javax.servlet - servlet-api - - - org.mortbay.jetty - servlet-api-2.5 - - - org.mortbay.jetty - servlet-api - - - - - ${project.groupId} - titan-hadoop-core - ${project.version} - - - ${project.groupId} - titan-hadoop-core - ${project.version} - tests - test - - - ${project.groupId} - titan-hadoop-core - ${project.version} - shared-resources - test - true - - - org.apache.hbase - hbase-client - ${hbase098.core.version}-hadoop1 - true - test - - - - org.jruby - jruby-complete - - - - - org.apache.mrunit - mrunit - ${mrunit.version} - hadoop1 - - - - ${project.groupId} - titan-es - ${project.version} - tests - test - - - - - - - maven-assembly-plugin - - - maven-dependency-plugin - - - maven-failsafe-plugin - - - maven-surefire-plugin - - - - diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java deleted file mode 100644 index 4432044208..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/DistCacheConfigurer.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.hadoop.config.job.AbstractDistCacheConfigurer; -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Job; - -import java.io.IOException; - -public class DistCacheConfigurer extends AbstractDistCacheConfigurer implements JobClasspathConfigurer { - - public DistCacheConfigurer(String mapredJarFilename) { - super(mapredJarFilename); - } - - @Override - public void configure(Job job) throws IOException { - - for (Path p : getLocalPaths()) { - Configuration conf = job.getConfiguration(); - FileSystem jobFS = FileSystem.get(conf); - FileSystem localFS = FileSystem.getLocal(conf); - Path stagedPath = uploadFileIfNecessary(localFS, p, jobFS); - DistributedCache.addFileToClassPath(stagedPath, conf, jobFS); - } - - // We don't really need to set a mapred job jar here, - // but doing so suppresses a warning - String mj = getMapredJar(); - if (null != mj) - job.getConfiguration().set(Hadoop1Compat.CFG_JOB_JAR, mj); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java deleted file mode 100644 index b47025a882..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1Compat.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.core.TitanException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics; -import com.thinkaurelius.titan.graphdb.configuration.TitanConstants; -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import com.thinkaurelius.titan.hadoop.formats.cassandra.CassandraBinaryInputFormat; -import com.thinkaurelius.titan.hadoop.scan.HadoopVertexScanMapper; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.*; -import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; - -import com.thinkaurelius.titan.hadoop.compat.HadoopCompat; - -import java.io.IOException; - -public class Hadoop1Compat implements HadoopCompat { - - static final String CFG_SPECULATIVE_MAPS = "mapred.map.tasks.speculative.execution"; - static final String CFG_SPECULATIVE_REDUCES = "mapred.reduce.tasks.speculative.execution"; - static final String CFG_JOB_JAR = "mapred.jar"; - - @Override - public TaskAttemptContext newTask(Configuration c, TaskAttemptID t) { - return new TaskAttemptContext(c, t); - } - - @Override - public String getSpeculativeMapConfigKey() { - return CFG_SPECULATIVE_MAPS; - } - - @Override - public String getSpeculativeReduceConfigKey() { - return CFG_SPECULATIVE_REDUCES; - } - - @Override - public String getMapredJarConfigKey() { - return CFG_JOB_JAR; - } - - @Override - public long getContextCounter(TaskInputOutputContext context, String group, String name) { - return context.getCounter(group, name).getValue(); - } - - @Override - public void incrementContextCounter(TaskInputOutputContext context, - String group, String name, long incr) { - context.getCounter(group, name).increment(incr); - } - - @Override - public Configuration getContextConfiguration(TaskAttemptContext context) { - return context.getConfiguration(); - } - - @Override - public JobClasspathConfigurer newMapredJarConfigurer(String mapredJarPath) { - return new MapredJarConfigurer(mapredJarPath); - } - - @Override - public JobClasspathConfigurer newDistCacheConfigurer() { - return new DistCacheConfigurer("titan-hadoop-core-" + TitanConstants.VERSION + ".jar"); - } - - @Override - public Configuration getJobContextConfiguration(JobContext context) { - return context.getConfiguration(); - } - - @Override - public Configuration newImmutableConfiguration(Configuration base) { - return new ImmutableConfiguration(base); - } - - @Override - public ScanMetrics getMetrics(Counters c) { - return new Hadoop1CountersScanMetrics(c); - } - - @Override - public String getJobFailureString(Job j) { - return j.toString(); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java deleted file mode 100644 index a9c236d3fc..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/Hadoop1CountersScanMetrics.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics; -import com.thinkaurelius.titan.hadoop.scan.HadoopContextScanMetrics; -import org.apache.hadoop.mapreduce.Counters; - -public class Hadoop1CountersScanMetrics implements ScanMetrics { - - private final Counters counters; - - public Hadoop1CountersScanMetrics(Counters counters) { - this.counters = counters; - } - - @Override - public long getCustom(String metric) { - return counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).getValue(); - } - - @Override - public void incrementCustom(String metric, long delta) { - counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).increment(delta); - } - - @Override - public void incrementCustom(String metric) { - incrementCustom(metric, 1L); - } - - @Override - public long get(Metric metric) { - return counters.getGroup(HadoopContextScanMetrics.STANDARD_COUNTER_GROUP).findCounter(metric.name()).getValue(); - } - - @Override - public void increment(Metric metric) { - counters.getGroup(HadoopContextScanMetrics.STANDARD_COUNTER_GROUP).findCounter(metric.name()).increment(1L); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java deleted file mode 100644 index b85647f018..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/ImmutableConfiguration.java +++ /dev/null @@ -1,283 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; - -import java.io.*; -import java.net.URL; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -public class ImmutableConfiguration extends Configuration { - - private final Configuration encapsulated; - - public ImmutableConfiguration(final Configuration encapsulated) { - this.encapsulated = encapsulated; - } - - public static void addDefaultResource(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(URL url) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(Path file) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void addResource(InputStream in) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void reloadConfiguration() { - //throw new UnsupportedOperationException("This configuration instance is immutable"); - encapsulated.reloadConfiguration(); // allowed to simplify testing - } - - @Override - public String get(String name) { - return encapsulated.get(name); - } - - @Override - public String getRaw(String name) { - return encapsulated.getRaw(name); - } - - @Override - public void set(String name, String value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void unset(String name) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void setIfUnset(String name, String value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public String get(String name, String defaultValue) { - return encapsulated.get(name, defaultValue); - } - - @Override - public int getInt(String name, int defaultValue) { - return encapsulated.getInt(name, defaultValue); - } - - @Override - public void setInt(String name, int value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public long getLong(String name, long defaultValue) { - return encapsulated.getLong(name, defaultValue); - } - - @Override - public void setLong(String name, long value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public float getFloat(String name, float defaultValue) { - return encapsulated.getFloat(name, defaultValue); - } - - @Override - public void setFloat(String name, float value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public boolean getBoolean(String name, boolean defaultValue) { - return encapsulated.getBoolean(name, defaultValue); - } - - @Override - public void setBoolean(String name, boolean value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public void setBooleanIfUnset(String name, boolean value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public > void setEnum(String name, T value) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public > T getEnum(String name, T defaultValue) { - return encapsulated.getEnum(name, defaultValue); - } - - @Override - public IntegerRanges getRange(String name, String defaultValue) { - return encapsulated.getRange(name, defaultValue); - } - - @Override - public Collection getStringCollection(String name) { - return encapsulated.getStringCollection(name); - } - - @Override - public String[] getStrings(String name) { - return encapsulated.getStrings(name); - } - - @Override - public String[] getStrings(String name, String... defaultValue) { - return encapsulated.getStrings(name, defaultValue); - } - - @Override - public void setStrings(String name, String... values) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Class getClassByName(String name) throws ClassNotFoundException { - return encapsulated.getClassByName(name); - } - - @Override - public Class[] getClasses(String name, Class... defaultValue) { - return encapsulated.getClasses(name, defaultValue); - } - - @Override - public Class getClass(String name, Class defaultValue) { - return encapsulated.getClass(name, defaultValue); - } - - @Override - public Class getClass(String name, Class defaultValue, Class xface) { - return encapsulated.getClass(name, defaultValue, xface); - } - - @Override - public List getInstances(String name, Class xface) { - return encapsulated.getInstances(name, xface); - } - - @Override - public void setClass(String name, Class theClass, Class xface) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Path getLocalPath(String dirsProp, String path) throws IOException { - return encapsulated.getLocalPath(dirsProp, path); - } - - @Override - public File getFile(String dirsProp, String path) throws IOException { - return encapsulated.getFile(dirsProp, path); - } - - @Override - public URL getResource(String name) { - return encapsulated.getResource(name); - } - - @Override - public InputStream getConfResourceAsInputStream(String name) { - return encapsulated.getConfResourceAsInputStream(name); - } - - @Override - public Reader getConfResourceAsReader(String name) { - return encapsulated.getConfResourceAsReader(name); - } - - @Override - public int size() { - return encapsulated.size(); - } - - @Override - public void clear() { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public Iterator> iterator() { - return encapsulated.iterator(); - } - - @Override - public void writeXml(OutputStream out) throws IOException { - encapsulated.writeXml(out); - } - - @Override - public void writeXml(Writer out) throws IOException { - encapsulated.writeXml(out); - } - - public static void dumpConfiguration(Configuration config, Writer out) throws IOException { - Configuration.dumpConfiguration(config, out); - } - - @Override - public ClassLoader getClassLoader() { - return encapsulated.getClassLoader(); - } - - @Override - public void setClassLoader(ClassLoader classLoader) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - @Override - public String toString() { - return encapsulated.toString(); - } - - @Override - public void setQuietMode(boolean quietmode) { - throw new UnsupportedOperationException("This configuration instance is immutable"); - } - - public static void main(String[] args) throws Exception { - Configuration.main(args); - } - - @Override - public void readFields(DataInput in) throws IOException { - encapsulated.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - encapsulated.write(out); - } - - @Override - public Map getValByRegex(String regex) { - return encapsulated.getValByRegex(regex); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java deleted file mode 100644 index 2be1c5f013..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/compat/h1/MapredJarConfigurer.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.thinkaurelius.titan.hadoop.compat.h1; - -import com.thinkaurelius.titan.hadoop.config.job.JobClasspathConfigurer; -import org.apache.hadoop.mapreduce.Job; - -import java.io.IOException; - -public class MapredJarConfigurer implements JobClasspathConfigurer { - - private final String mapredJar; - - public MapredJarConfigurer(String mapredJar) { - this.mapredJar = mapredJar; - } - - @Override - public void configure(Job job) throws IOException { - job.getConfiguration().set(Hadoop1Compat.CFG_JOB_JAR, mapredJar); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java deleted file mode 100644 index 2134ed222d..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputCommitter.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import java.io.IOException; - -public class TitanH1OutputCommitter extends OutputCommitter { - private final TitanH1OutputFormat tof; - - public TitanH1OutputCommitter(TitanH1OutputFormat tof) { - this.tof = tof; - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - - } - - @Override - public void setupTask(TaskAttemptContext taskContext) throws IOException { - - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { - return tof.hasModifications(taskContext.getTaskAttemptID()); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) throws IOException { - tof.commit(taskContext.getTaskAttemptID()); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) throws IOException { - tof.abort(taskContext.getTaskAttemptID()); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java deleted file mode 100644 index a898ed5600..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1OutputFormat.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.thinkaurelius.titan.core.TitanFactory; -import com.thinkaurelius.titan.graphdb.database.StandardTitanGraph; -import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx; -import com.thinkaurelius.titan.hadoop.config.ModifiableHadoopConfiguration; -import com.thinkaurelius.titan.hadoop.config.TitanHadoopConfiguration; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; -import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil; -import org.apache.tinkerpop.gremlin.process.computer.VertexProgram; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -public class TitanH1OutputFormat extends OutputFormat { - - private static final Logger log = LoggerFactory.getLogger(TitanH1OutputFormat.class); - - private final ConcurrentMap transactions = new ConcurrentHashMap<>(); - - private StandardTitanGraph graph; - - private Set persistableKeys; - - @Override - public RecordWriter getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - - synchronized (this) { - if (null == graph) { - Configuration hadoopConf = taskAttemptContext.getConfiguration(); - ModifiableHadoopConfiguration mhc = - ModifiableHadoopConfiguration.of(TitanHadoopConfiguration.MAPRED_NS, hadoopConf); - graph = (StandardTitanGraph) TitanFactory.open(mhc.getTitanGraphConf()); - } - } - - // Special case for a TP3 vertex program: persist only those properties whose keys are - // returned by VertexProgram.getComputeKeys() - if (null == persistableKeys) { - try { - persistableKeys = VertexProgram.createVertexProgram(graph, - ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration())).getElementComputeKeys(); - log.debug("Set persistableKeys={}", Joiner.on(",").join(persistableKeys)); - } catch (Exception e) { - log.debug("Unable to detect or instantiate vertex program", e); - persistableKeys = ImmutableSet.of(); - } - } - - StandardTitanTx tx = transactions.computeIfAbsent(taskAttemptContext.getTaskAttemptID(), - id -> (StandardTitanTx)graph.newTransaction()); - return new TitanH1RecordWriter(taskAttemptContext, tx, persistableKeys); - } - - @Override - public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException { - // TODO check output configuration for minimum set of keys here? - } - - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws IOException, - InterruptedException { - return new TitanH1OutputCommitter(this); - } - - void commit(TaskAttemptID id) { - StandardTitanTx tx = transactions.remove(id); - if (null == tx) { - log.warn("Detected concurrency in task commit"); - return; - } - tx.commit(); - } - - void abort(TaskAttemptID id) { - StandardTitanTx tx = transactions.remove(id); - if (null == tx) { - log.warn("Detected concurrency in task abort"); - return; - } - tx.rollback(); - } - - boolean hasModifications(TaskAttemptID id) { - StandardTitanTx tx = transactions.get(id); - // if tx is null, something is horribly wrong - return tx.hasModifications(); - } -} diff --git a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java b/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java deleted file mode 100644 index b2f4f28e8b..0000000000 --- a/titan-hadoop-parent/titan-hadoop-1/src/main/java/com/thinkaurelius/titan/hadoop/formats/TitanH1RecordWriter.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.thinkaurelius.titan.hadoop.formats; - -import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; -import org.apache.tinkerpop.gremlin.structure.Vertex; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Set; - -public class TitanH1RecordWriter extends RecordWriter { - - private static final Logger log = LoggerFactory.getLogger(TitanH1RecordWriter.class); - - private final TaskAttemptContext taskAttemptContext; - private final StandardTitanTx tx; - private final Set persistableKeys; - - public TitanH1RecordWriter(TaskAttemptContext taskAttemptContext, StandardTitanTx tx, Set persistableKeys) { - this.taskAttemptContext = taskAttemptContext; - this.tx = tx; - this.persistableKeys = persistableKeys; - } - - @Override - public void write(NullWritable key, VertexWritable value) throws IOException, InterruptedException { - // TODO tolerate possibility that concurrent OLTP activity has deleted the vertex? maybe configurable... - Object vertexID = value.get().id(); - Vertex vertex = tx.vertices(vertexID).next(); - Iterator> vpIter = value.get().properties(); - while (vpIter.hasNext()) { - VertexProperty vp = vpIter.next(); - if (!persistableKeys.isEmpty() && !persistableKeys.contains(vp.key())) { - log.debug("[vid {}] skipping key {}", vertexID, vp.key()); - continue; - } - vertex.property(vp.key(), vp.value()); - log.debug("[vid {}] set {}={}", vertexID, vp.key(), vp.value()); - } - } - - @Override - public void close(TaskAttemptContext context) throws IOException, InterruptedException { - // commit/rollback happens in TitanOutputCommitter - // nothing to do here - } -} diff --git a/titan-hadoop-parent/titan-hadoop/pom.xml b/titan-hadoop-parent/titan-hadoop/pom.xml index c0a5e831f0..38138d2dfe 100644 --- a/titan-hadoop-parent/titan-hadoop/pom.xml +++ b/titan-hadoop-parent/titan-hadoop/pom.xml @@ -22,12 +22,6 @@ ${project.version} true - - ${project.groupId} - titan-hadoop-1 - ${project.version} - true - ${project.groupId} titan-hadoop-2 diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 31d00b0445..9c0b041917 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -15,6 +15,7 @@ titan-hbase-core titan-hbase-098 titan-hbase-10 + titan-hbase-11 titan-hbase diff --git a/titan-hbase-parent/titan-hbase-11/pom.xml b/titan-hbase-parent/titan-hbase-11/pom.xml new file mode 100644 index 0000000000..40be33c02e --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/pom.xml @@ -0,0 +1,167 @@ + + 4.0.0 + + com.thinkaurelius.titan + titan-hbase-parent + 1.0.1-SNAPSHOT + ../pom.xml + + titan-hbase-11 + Titan-HBase: 1.1 Compatibility Shim + http://thinkaurelius.github.com/titan/ + + + 1.6.8 + ${project.parent.parent.basedir} + + + + + ${project.groupId} + titan-hbase-core + ${project.version} + + + commons-net + commons-net + + + + + org.apache.hbase + hbase-client + ${hbase111.version} + true + + + avro + org.apache.avro + + + jruby-complete + org.jruby + + + asm + asm + + + + + org.apache.hbase + hbase-server + ${hbase111.version} + true + + + avro + org.apache.avro + + + jruby-complete + org.jruby + + + asm + asm + + + + + jruby-complete + org.jruby + ${jruby.version} + test + + + + + src/main/java + ../common/test/java + + + ${project.basedir}/../common/test/resources + true + + + + + maven-resources-plugin + + + filter-hbase-conf + + + + + maven-dependency-plugin + + + generate-cached-classpath + + + unpack-common-test-classes + + + + + maven-assembly-plugin + + + copy-hbase-bin + + + + + maven-clean-plugin + + + clean-hbase-files + + + + + maven-deploy-plugin + + + default-deploy + none + + + + + + + + + hadoop2 + + + + !hadoop.profile + + + + + + org.apache.hadoop + hadoop-annotations + ${hadoop2.version} + true + + + org.apache.hadoop + hadoop-hdfs + ${hadoop2.version} + true + + + org.apache.hadoop + hadoop-client + ${hadoop2.version} + true + + + + + diff --git a/titan-hbase-parent/titan-hbase-11/readme.txt b/titan-hbase-parent/titan-hbase-11/readme.txt new file mode 100644 index 0000000000..4560298aa1 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/readme.txt @@ -0,0 +1 @@ +request to add this tree in main titan via pull request #1219 diff --git a/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java new file mode 100644 index 0000000000..86b928bb2c --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_1.java @@ -0,0 +1,127 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.thinkaurelius.titan.util.system.IOUtils; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; + +public class HBaseAdmin1_1 implements AdminMask +{ + + private static final Logger log = LoggerFactory.getLogger(HBaseAdmin1_1.class); + + private final Admin adm; + + public HBaseAdmin1_1(HBaseAdmin adm) + { + this.adm = adm; + } + @Override + public void clearTable(String tableString, long timestamp) throws IOException + { + TableName tableName = TableName.valueOf(tableString); + + if (!adm.tableExists(tableName)) { + log.debug("Attempted to clear table {} before it exists (noop)", tableString); + return; + } + + if (!adm.isTableDisabled(tableName)) + adm.disableTable(tableName); + + if (!adm.isTableDisabled(tableName)) + throw new RuntimeException("Unable to disable table " + tableName); + + // This API call appears to both truncate and reenable the table. + log.info("Truncating table {}", tableName); + adm.truncateTable(tableName, true /* preserve splits */); + + try { + adm.enableTable(tableName); + } catch (TableNotDisabledException e) { + // This triggers seemingly every time in testing with 1.0.2. + log.debug("Table automatically reenabled by truncation: {}", tableName, e); + } + } + + @Override + public HTableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException + { + return adm.getTableDescriptor(TableName.valueOf(tableString)); + } + + @Override + public boolean tableExists(String tableString) throws IOException + { + return adm.tableExists(TableName.valueOf(tableString)); + } + + @Override + public void createTable(HTableDescriptor desc) throws IOException + { + adm.createTable(desc); + } + + @Override + public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException + { + adm.createTable(desc, startKey, endKey, numRegions); + } + + @Override + public int getEstimatedRegionServerCount() + { + int serverCount = -1; + try { + serverCount = adm.getClusterStatus().getServers().size(); + log.debug("Read {} servers from HBase ClusterStatus", serverCount); + } catch (IOException e) { + log.debug("Unable to retrieve HBase cluster status", e); + } + return serverCount; + } + + @Override + public void disableTable(String tableString) throws IOException + { + adm.disableTable(TableName.valueOf(tableString)); + } + + @Override + public void enableTable(String tableString) throws IOException + { + adm.enableTable(TableName.valueOf(tableString)); + } + + @Override + public boolean isTableDisabled(String tableString) throws IOException + { + return adm.isTableDisabled(TableName.valueOf(tableString)); + } + + @Override + public void addColumn(String tableString, HColumnDescriptor columnDescriptor) throws IOException + { + adm.addColumn(TableName.valueOf(tableString), columnDescriptor); + } + + @Override + public void close() throws IOException + { + adm.close(); + } +} diff --git a/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java new file mode 100644 index 0000000000..a25999f9d1 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java @@ -0,0 +1,45 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.io.compress.Compression; + +public class HBaseCompat1_1 implements HBaseCompat { + + @Override + public void setCompression(HColumnDescriptor cd, String algo) { + cd.setCompressionType(Compression.Algorithm.valueOf(algo)); + } + + @Override + public HTableDescriptor newTableDescriptor(String tableName) { + TableName tn = TableName.valueOf(tableName); + return new HTableDescriptor(tn); + } + + @Override + public ConnectionMask createConnection(Configuration conf) throws IOException + { + return new HConnection1_1(ConnectionFactory.createConnection(conf)); + } + + @Override + public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc) + { + tdesc.addFamily(cdesc); + } + + @Override + public void setTimestamp(Delete d, long timestamp) + { + d.setTimestamp(timestamp); + } + +} diff --git a/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java new file mode 100644 index 0000000000..12df7f4686 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_1.java @@ -0,0 +1,36 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.HBaseAdmin; + +public class HConnection1_1 implements ConnectionMask +{ + + private final Connection cnx; + + public HConnection1_1(Connection cnx) + { + this.cnx = cnx; + } + + @Override + public TableMask getTable(String name) throws IOException + { + return new HTable1_1(cnx.getTable(TableName.valueOf(name))); + } + + @Override + public AdminMask getAdmin() throws IOException + { + return new HBaseAdmin1_1(new HBaseAdmin(cnx)); + } + + @Override + public void close() throws IOException + { + cnx.close(); + } +} diff --git a/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java new file mode 100644 index 0000000000..02bc06e2db --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_1.java @@ -0,0 +1,47 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Row; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; + +public class HTable1_1 implements TableMask +{ + private final Table table; + + public HTable1_1(Table table) + { + this.table = table; + } + + @Override + public ResultScanner getScanner(Scan filter) throws IOException + { + return table.getScanner(filter); + } + + @Override + public Result[] get(List gets) throws IOException + { + return table.get(gets); + } + + @Override + public void batch(List writes, Object[] results) throws IOException, InterruptedException + { + table.batch(writes, results); + /* table.flushCommits(); not needed anymore */ + } + + @Override + public void close() throws IOException + { + table.close(); + } +} diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb new file mode 100755 index 0000000000..5bcb5b6b60 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/draining_servers.rb @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Add or remove servers from draining mode via zookeeper + +require 'optparse' +include Java + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "draining_servers" + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: ./hbase org.jruby.Main #{NAME}.rb [options] add|remove|list || ..." + opts.separator 'Add remove or list servers in draining mode. Can accept either hostname to drain all region servers' + + 'in that host, a host:port pair or a host,port,startCode triplet. More than one server can be given separated by space' + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end +end +optparse.parse! + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +def getServerNames(hostOrServers, config) + ret = [] + + for hostOrServer in hostOrServers + # check whether it is already serverName. No need to connect to cluster + parts = hostOrServer.split(',') + if parts.size() == 3 + ret << hostOrServer + else + admin = HBaseAdmin.new(config) if not admin + servers = getServers(admin) + + hostOrServer = hostOrServer.gsub(/:/, ",") + for server in servers + ret << server if server.start_with?(hostOrServer) + end + end + end + + admin.close() if admin + return ret +end + +def addServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.createAndFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +def removeServers(options, hostOrServers) + config = HBaseConfiguration.create() + servers = getServerNames(hostOrServers, config) + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + begin + for server in servers + node = ZKUtil.joinZNode(parentZnode, server) + ZKUtil.deleteNodeFailSilent(zkw, node) + end + ensure + zkw.close() + end +end + +# list servers in draining mode +def listServers(options) + config = HBaseConfiguration.create() + + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(config, "draining_servers", nil) + parentZnode = zkw.drainingZNode + + servers = ZKUtil.listChildrenNoWatch(zkw, parentZnode) + servers.each {|server| puts server} +end + +hostOrServers = ARGV[1..ARGV.size()] + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase") + logger.setLevel(org.apache.log4j.Level::WARN) + logger = org.apache.log4j.Logger.getLogger("org.apache.zookeeper") + logger.setLevel(org.apache.log4j.Level::WARN) + end + return apacheLogger +end + +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'add' + if ARGV.length < 2 + puts optparse + exit 1 + end + addServers(options, hostOrServers) + when 'remove' + if ARGV.length < 2 + puts optparse + exit 1 + end + removeServers(options, hostOrServers) + when 'list' + listServers(options) + else + puts optparse + exit 3 +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb new file mode 100755 index 0000000000..f7e1ff6793 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/get-active-master.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env hbase-jruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints the hostname of the machine running the active master. + +include Java +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.ServerName +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher + +# disable debug/info logging on this script for clarity +log_level = org.apache.log4j.Level::ERROR +org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) +org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) + +config = HBaseConfiguration.create + +zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) +begin + master_address = ZKUtil.getData(zk, zk.masterAddressZNode) + if master_address + puts ServerName.parseFrom(master_address).getHostname() + else + puts 'Master not running' + end +ensure + zk.close() +end + diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh new file mode 100755 index 0000000000..98b27e31b5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/graceful_stop.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Move regions off a server then stop it. Optionally restart and reload. +# Turn off the balancer before running this script. +function usage { + echo "Usage: graceful_stop.sh [--config ] [-d] [-e] [--restart [--reload]] [--thrift] [--rest] " + echo " thrift If we should stop/start thrift before/after the hbase stop/start" + echo " rest If we should stop/start rest before/after the hbase stop/start" + echo " restart If we should restart after graceful stop" + echo " reload Move offloaded regions back on to the restarted server" + echo " d|debug Print helpful debug information" + echo " maxthreads xx Limit the number of threads used by the region mover. Default value is 1." + echo " hostname Hostname of server we are to stop" + echo " e|failfast Set -e so exit immediately if any command exits with non-zero status" + exit 1 +} + +if [ $# -lt 1 ]; then + usage +fi + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh +# Get arguments +restart= +reload= +debug= +thrift= +rest= +maxthreads=1 +failfast= +while [ $# -gt 0 ] +do + case "$1" in + --thrift) thrift=true; shift;; + --rest) rest=true; shift;; + --restart) restart=true; shift;; + --reload) reload=true; shift;; + --failfast | -e) failfast=true; shift;; + --debug | -d) debug="--debug"; shift;; + --maxthreads) shift; maxthreads=$1; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# "$@" contains the rest. Must be at least the hostname left. +if [ $# -lt 1 ]; then + usage +fi + +# Emit a log line w/ iso8901 date prefixed +log() { + echo `date +%Y-%m-%dT%H:%M:%S` $1 +} + +# See if we should set fail fast before we do anything. +if [ "$failfast" != "" ]; then + log "Set failfast, will exit immediately if any command exits with non-zero status" + set -e +fi + +hostname=$1 +filename="/tmp/$hostname" + +local=false +localhostname=`/bin/hostname` + +if [ "$localhostname" == "$hostname" ]; then + local=true +fi + +log "Disabling load balancer" +HBASE_BALANCER_STATE=`echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell | tail -3 | head -1` +log "Previous balancer state was $HBASE_BALANCER_STATE" + +log "Unloading $hostname region(s)" +HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads unload $hostname +log "Unloaded $hostname region(s)" + +# Stop the server(s). Have to put hostname into its own little file for hbase-daemons.sh +hosts="/tmp/$(basename $0).$$.tmp" +echo $hostname >> $hosts +if [ "$thrift" != "" ]; then + log "Stopping thrift" + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop thrift + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift + fi +fi +if [ "$rest" != "" ]; then + log "Stopping rest" + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop rest + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest + fi +fi +log "Stopping regionserver" +if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} stop regionserver +else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver +fi +if [ "$restart" != "" ]; then + log "Restarting regionserver" + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start regionserver + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver + fi + if [ "$thrift" != "" ]; then + log "Restarting thrift" + # -b 0.0.0.0 says listen on all interfaces rather than just default. + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start thrift -b 0.0.0.0 + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 + fi + fi + if [ "$rest" != "" ]; then + log "Restarting rest" + if [ "$local" ]; then + "$bin"/hbase-daemon.sh --config ${HBASE_CONF_DIR} start rest + else + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest + fi + fi + if [ "$reload" != "" ]; then + log "Reloading $hostname region(s)" + HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug --maxthreads=$maxthreads load $hostname + log "Reloaded $hostname region(s)" + fi +fi + +# Restore balancer state +if [ $HBASE_BALANCER_STATE != "false" ]; then + log "Restoring balancer state to " $HBASE_BALANCER_STATE + echo "balance_switch $HBASE_BALANCER_STATE" | "$bin"/hbase --config ${HBASE_CONF_DIR} shell &> /dev/null +fi + +# Cleanup tmp files. +trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase new file mode 100755 index 0000000000..6ad7a91097 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase @@ -0,0 +1,421 @@ +#! /usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# The hbase command script. Based on the hadoop command script putting +# in hbase classes, libs and configurations ahead of hadoop's. +# +# TODO: Narrow the amount of duplicated code. +# +# Environment Variables: +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HBASE_CLASSPATH Extra Java CLASSPATH entries. +# +# HBASE_CLASSPATH_PREFIX Extra Java CLASSPATH entries that should be +# prefixed to the system classpath. +# +# HBASE_HEAPSIZE The maximum amount of heap to use. +# Default is unset and uses the JVMs default setting +# (usually 1/4th of the available memory). +# +# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding +# native libraries. +# +# HBASE_OPTS Extra Java runtime options. +# +# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. +# +# HBASE_ROOT_LOGGER The root appender. Default is INFO,console +# +# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. +# Defaults to the jar packaged with HBase. +# +# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. +# Empty by default. +# +# HBASE_SHELL_OPTS Extra options passed to the hbase shell. +# Empty by default. +# +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh + +cygwin=false +case "`uname`" in +CYGWIN*) cygwin=true;; +esac + +# Detect if we are in hbase sources dir +in_dev_env=false +if [ -d "${HBASE_HOME}/target" ]; then + in_dev_env=true +fi + +# if no args specified, show usage +if [ $# = 0 ]; then + echo "Usage: hbase [] []" + echo "Options:" + echo " --config DIR Configuration direction to use. Default: ./conf" + echo " --hosts HOSTS Override the list in 'regionservers' file" + echo "" + echo "Commands:" + echo "Some commands take arguments. Pass no args or -h for usage." + echo " shell Run the HBase shell" + echo " hbck Run the hbase 'fsck' tool" + echo " snapshot Create a new snapshot of a table" + echo " wal Write-ahead-log analyzer" + echo " hfile Store file analyzer" + echo " zkcli Run the ZooKeeper shell" + echo " upgrade Upgrade hbase" + echo " master Run an HBase HMaster node" + echo " regionserver Run an HBase HRegionServer node" + echo " zookeeper Run a Zookeeper server" + echo " rest Run an HBase REST server" + echo " thrift Run the HBase Thrift server" + echo " thrift2 Run the HBase Thrift2 server" + echo " clean Run the HBase clean up script" + echo " classpath Dump hbase CLASSPATH" + echo " mapredcp Dump CLASSPATH entries required by mapreduce" + echo " pe Run PerformanceEvaluation" + echo " ltt Run LoadTestTool" + echo " version Print the version" + echo " CLASSNAME Run the class named CLASSNAME" + exit 1 +fi + +# get arguments +COMMAND=$1 +shift + +JAVA=$JAVA_HOME/bin/java + +# override default settings for this command, if applicable +if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then + . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" +fi + +add_size_suffix() { + # add an 'm' suffix if the argument is missing one, otherwise use whats there + local val="$1" + local lastchar=${val: -1} + if [[ "mMgG" == *$lastchar* ]]; then + echo $val + else + echo ${val}m + fi +} + +if [[ -n "$HBASE_HEAPSIZE" ]]; then + JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_HEAPSIZE)" +fi + +if [[ -n "$HBASE_OFFHEAPSIZE" ]]; then + JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)" +fi + +# so that filenames w/ spaces are handled correctly in loops below +ORIG_IFS=$IFS +IFS= + +# CLASSPATH initially contains $HBASE_CONF_DIR +CLASSPATH="${HBASE_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +add_to_cp_if_exists() { + if [ -d "$@" ]; then + CLASSPATH=${CLASSPATH}:"$@" + fi +} + +# For releases, add hbase & webapps to CLASSPATH +# Webapps must come first else it messes up Jetty +if [ -d "$HBASE_HOME/hbase-webapps" ]; then + add_to_cp_if_exists "${HBASE_HOME}" +fi +#add if we are in a dev environment +if [ -d "$HBASE_HOME/hbase-server/target/hbase-webapps" ]; then + add_to_cp_if_exists "${HBASE_HOME}/hbase-server/target" +fi + +add_maven_deps_to_classpath() { + f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ] + then + echo "As this is a development environment, we need ${f} to be generated from maven (command: mvn install -DskipTests)" + exit 1 + fi + CLASSPATH=${CLASSPATH}:`cat "${f}"` +} + + +add_maven_deps_to_classpath() { + f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ]; then + # ------------------- + # Titan build support + # ------------------- + # Check for Maven command line options file. + # If titan-hbase has already been built, then this file will contain options + # to target the version of Hadoop used during compilation. This is + # necessary when the user compiles titan-hbase with a non-default Hadoop + # version (e.g. 2.2.0). If we omitted this check, then we'd try to start + # HBase with the default Hadoop version (currentl 1.0.4), and it would break. + MVN_OPTS= + optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" + unset IFS + if [ -f "${optf}" ]; then + MVN_OPTS=`cat "${optf}"` + fi + mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + IFS= + fi + CLASSPATH=${CLASSPATH}:`cat "${f}"` +} + + +#Add the development env class path stuff +if $in_dev_env; then + add_maven_deps_to_classpath +fi + +#add the hbase jars for each module +for f in $HBASE_HOME/hbase-jars/hbase*.jar; do + if [[ $f = *sources.jar ]] + then + : # Skip sources.jar + elif [ -f $f ] + then + CLASSPATH=${CLASSPATH}:$f; + fi +done + +# Add libs to CLASSPATH +for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# default log directory & file +if [ "$HBASE_LOG_DIR" = "" ]; then + HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +if [ "$HBASE_LOGFILE" = "" ]; then + HBASE_LOGFILE='hbase.log' +fi + +function append_path() { + if [ -z "$1" ]; then + echo $2 + else + echo $1:$2 + fi +} + +JAVA_PLATFORM="" + +# if HBASE_LIBRARY_PATH is defined lets use it as first or second option +if [ "$HBASE_LIBRARY_PATH" != "" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") +fi + +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +# Allow this functionality to be disabled +if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then + HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) + if [ -f ${HADOOP_IN_PATH} ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") + fi + CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) + fi +fi + +# Add user-specified CLASSPATH last +if [ "$HBASE_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} +fi + +# Add user-specified CLASSPATH prefix first +if [ "$HBASE_CLASSPATH_PREFIX" != "" ]; then + CLASSPATH=${HBASE_CLASSPATH_PREFIX}:${CLASSPATH} +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HBASE_HOME=`cygpath -d "$HBASE_HOME"` + HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` +fi + +if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then + if [ -z $JAVA_PLATFORM ]; then + JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` + fi + if [ -d "$HBASE_HOME/build/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + fi + + if [ -d "${HBASE_HOME}/lib/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + fi +fi + +# cygwin path translation +if $cygwin; then + JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` +fi + +# restore ordinary behaviour +unset IFS + +#Set the right GC options based on the what we are running +declare -a server_cmds=("master" "regionserver" "thrift" "thrift2" "rest" "avro" "zookeeper") +for cmd in ${server_cmds[@]}; do + if [[ $cmd == $COMMAND ]]; then + server=true + break + fi +done + +if [[ $server ]]; then + HBASE_OPTS="$HBASE_OPTS $SERVER_GC_OPTS" +else + HBASE_OPTS="$HBASE_OPTS $CLIENT_GC_OPTS" +fi + +# figure out which class to run +if [ "$COMMAND" = "shell" ] ; then + # eg export JRUBY_HOME=/usr/local/share/jruby + if [ "$JRUBY_HOME" != "" ] ; then + CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" + HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" + fi + # find the hbase ruby sources + HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" + CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" +elif [ "$COMMAND" = "hbck" ] ; then + CLASS='org.apache.hadoop.hbase.util.HBaseFsck' +# TODO remove old 'hlog' version +elif [ "$COMMAND" = "hlog" -o "$COMMAND" = "wal" ] ; then + CLASS='org.apache.hadoop.hbase.wal.WALPrettyPrinter' +elif [ "$COMMAND" = "hfile" ] ; then + CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter' +elif [ "$COMMAND" = "zkcli" ] ; then + CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer" +elif [ "$COMMAND" = "upgrade" ] ; then + CLASS="org.apache.hadoop.hbase.migration.UpgradeTo96" +elif [ "$COMMAND" = "snapshot" ] ; then + CLASS="org.apache.hadoop.hbase.snapshot.CreateSnapshot" +elif [ "$COMMAND" = "master" ] ; then + CLASS='org.apache.hadoop.hbase.master.HMaster' + if [ "$1" != "stop" ] && [ "$1" != "clear" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" + fi +elif [ "$COMMAND" = "regionserver" ] ; then + CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" + fi +elif [ "$COMMAND" = "thrift" ] ; then + CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "thrift2" ] ; then + CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "rest" ] ; then + CLASS='org.apache.hadoop.hbase.rest.RESTServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" + fi +elif [ "$COMMAND" = "zookeeper" ] ; then + CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" + fi +elif [ "$COMMAND" = "clean" ] ; then + case $1 in + --cleanZk|--cleanHdfs|--cleanAll) + matches="yes" ;; + *) ;; + esac + if [ $# -ne 1 -o "$matches" = "" ]; then + echo "Usage: hbase clean (--cleanZk|--cleanHdfs|--cleanAll)" + echo "Options: " + echo " --cleanZk cleans hbase related data from zookeeper." + echo " --cleanHdfs cleans hbase related data from hdfs." + echo " --cleanAll cleans hbase related data from both zookeeper and hdfs." + exit 1; + fi + "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ + exit $? +elif [ "$COMMAND" = "mapredcp" ] ; then + CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' +elif [ "$COMMAND" = "classpath" ] ; then + echo $CLASSPATH + exit 0 +elif [ "$COMMAND" = "pe" ] ; then + CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' + HBASE_OPTS="$HBASE_OPTS $HBASE_PE_OPTS" +elif [ "$COMMAND" = "ltt" ] ; then + CLASS='org.apache.hadoop.hbase.util.LoadTestTool' + HBASE_OPTS="$HBASE_OPTS $HBASE_LTT_OPTS" +elif [ "$COMMAND" = "version" ] ; then + CLASS='org.apache.hadoop.hbase.util.VersionInfo' +else + CLASS=$COMMAND +fi + +# Have JVM dump heap if we run out of memory. Files will be 'launch directory' +# and are named like the following: java_pid21612.hprof. Apparently it doesn't +# 'cost' to have this flag enabled. Its a 1.6 flag only. See: +# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" +HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" +HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" +HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" +fi + +# Enable security logging on the master and regionserver only +if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,RFAS}" +else + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" +fi + +HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" +# Exec unless HBASE_NOEXEC is set. +export CLASSPATH +if [ "${HBASE_NOEXEC}" != "" ]; then + "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" +else + exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" +fi diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh new file mode 100755 index 0000000000..596e366a30 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-cleanup.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# cleans hbase related data from zookeeper and hdfs if no hbase process is alive. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# + +usage="Usage: hbase-cleanup.sh (--cleanZk|--cleanHdfs|--cleanAll|--cleanAcls)" + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh + +case $1 in + --cleanZk|--cleanHdfs|--cleanAll|--cleanAcls) + matches="yes" ;; + *) ;; +esac +if [ $# -ne 1 -o "$matches" = "" ]; then + echo $usage + exit 1; +fi + +format_option=$1; + +zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` +if [ "$zparent" == "null" ]; then zparent="/hbase"; fi + +hrootdir=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.rootdir` +if [ "$hrootdir" == "null" ]; then hrootdir="file:///tmp/hbase-${USER}/hbase"; fi + +check_for_znodes() { + command=$1; + case $command in + regionservers) + zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` + if [ "$zchild" == "null" ]; then zchild="rs"; fi + ;; + backupmasters) + zchild=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.backup.masters` + if [ "$zchild" == "null" ]; then zchild="backup-masters"; fi + ;; + esac + znodes=`"$bin"/hbase zkcli ls $zparent/$zchild 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` + if [ "$znodes" != "" ]; then + echo -n "ZNode(s) [${znodes}] of $command are not expired. Exiting without cleaning hbase data." + echo #force a newline + exit 1; + else + echo -n "All ZNode(s) of $command are expired." + fi + echo #force a newline +} + +execute_zk_command() { + command=$1; + "$bin"/hbase zkcli $command 2>&1 +} + +execute_hdfs_command() { + command=$1; + "$bin"/hbase org.apache.hadoop.fs.FsShell $command 2>&1 +} + +execute_clean_acls() { + command=$1; + "$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZkAclReset $command 2>&1 +} + +clean_up() { + case $1 in + --cleanZk) + execute_zk_command "rmr ${zparent}"; + ;; + --cleanHdfs) + execute_hdfs_command "-rmr ${hrootdir}" + ;; + --cleanAll) + execute_zk_command "rmr ${zparent}"; + execute_hdfs_command "-rmr ${hrootdir}" + ;; + --cleanAcls) + execute_clean_acls; + ;; + *) + ;; + esac +} + +check_znode_exists() { + command=$1 + "$bin"/hbase zkcli stat $command 2>&1 | grep "Node does not exist\|Connection refused" +} + +check_znode_exists $zparent +if [ $? -ne 0 ]; then + # make sure the online region server(s) znode(s) have been deleted before continuing + check_for_znodes regionservers + # make sure the backup master(s) znode(s) has been deleted before continuing + check_for_znodes backupmasters + # make sure the master znode has been deleted before continuing + zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` + if [ "$zmaster" == "null" ]; then zmaster="master"; fi + zmaster=$zparent/$zmaster + check_znode_exists $zmaster + if [ $? -ne 0 ]; then + echo -n "Master ZNode is not expired. Exiting without cleaning hbase data." + echo #force a new line + exit 1 + else + echo "Active Master ZNode also expired." + fi + echo #force a newline +else + echo "HBase parent znode ${zparent} does not exist." +fi + +# cleans zookeeper and/or hdfs data. +clean_up $format_option diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh new file mode 100755 index 0000000000..0a474f7304 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-common.sh @@ -0,0 +1,41 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +#Shared function to wait for a process end. Take the pid and the command name as parameters +waitForProcessEnd() { + pidKilled=$1 + commandName=$2 + processedAt=`date +%s` + while kill -0 $pidKilled > /dev/null 2>&1; + do + echo -n "." + sleep 1; + # if process persists more than $HBASE_STOP_TIMEOUT (default 1200 sec) no mercy + if [ $(( `date +%s` - $processedAt )) -gt ${HBASE_STOP_TIMEOUT:-1200} ]; then + break; + fi + done + # process still there : kill -9 + if kill -0 $pidKilled > /dev/null 2>&1; then + echo -n force stopping $commandName with kill -9 $pidKilled + $JAVA_HOME/bin/jstack -l $pidKilled > "$logout" 2>&1 + kill -9 $pidKilled > /dev/null 2>&1 + fi + # Add a CR after we're done w/ dots. + echo +} diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd new file mode 100755 index 0000000000..5c1f1860f5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.cmd @@ -0,0 +1,78 @@ +@rem/* +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem included in all the hbase scripts with source command +@rem should not be executable directly +@rem also should not be passed any arguments, since we need original $* +@rem Modelled after $HADOOP_HOME/bin/hadoop-env.sh. + +@rem Make sure java environment is set +@rem + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +if "%1" == "--config" ( + set HBASE_CONF_DIR=%2 + shift + shift +) + +@rem the root of the hbase installation +if not defined HBASE_HOME ( + set HBASE_HOME=%HBASE_BIN_PATH%\.. +) + +@rem Allow alternate hbase conf dir location. +if not defined HBASE_CONF_DIR ( + set HBASE_CONF_DIR=%HBASE_HOME%\conf +) + +@rem List of hbase regions servers. +if not defined HBASE_REGIONSERVERS ( + set HBASE_REGIONSERVERS=%HBASE_CONF_DIR%\regionservers +) + +@rem List of hbase secondary masters. +if not defined HBASE_BACKUP_MASTERS ( + set HBASE_BACKUP_MASTERS=%HBASE_CONF_DIR%\backup-masters +) + +@rem Source the hbase-env.sh. Will have JAVA_HOME defined. +if exist "%HBASE_CONF_DIR%\hbase-env.cmd" ( + call "%HBASE_CONF_DIR%\hbase-env.cmd" +) + +if not defined JAVA_HOME ( + echo Warning: JAVA_HOME environment variable is not set. Defaulting to c:\apps\java + set JAVA_HOME=c:\apps\java +) + +if not exist "%JAVA_HOME%\bin\java.exe" ( + echo Error: JAVA_HOME is incorrectly set or could not find java at the location %JAVA_HOME%\bin\ + exit /B 2 +) + +set JAVA="%JAVA_HOME%\bin\java" + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh new file mode 100755 index 0000000000..5c518b1dc8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-config.sh @@ -0,0 +1,136 @@ +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# included in all the hbase scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* +# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. + +# resolve links - "${BASH_SOURCE-$0}" may be a softlink + +this="${BASH_SOURCE-$0}" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin">/dev/null; pwd` +this="$bin/$script" + +# the root of the hbase installation +if [ -z "$HBASE_HOME" ]; then + export HBASE_HOME=`dirname "$this"`/.. +fi + +#check to see if the conf dir or hbase home are given as an optional arguments +while [ $# -gt 1 ] +do + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HBASE_CONF_DIR=$confdir + elif [ "--hosts" = "$1" ] + then + shift + hosts=$1 + shift + HBASE_REGIONSERVERS=$hosts + else + # Presume we are at end of options and break + break + fi +done + +# Allow alternate hbase conf dir location. +HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" +# List of hbase regions servers. +HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" +# List of hbase secondary masters. +HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" +# Thrift JMX opts +if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_THRIFT_JMX_OPTS" ]; then + HBASE_THRIFT_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10103" +fi +# Thrift opts +if [ -z "$HBASE_THRIFT_OPTS" ]; then + export HBASE_THRIFT_OPTS="$HBASE_THRIFT_JMX_OPTS" +fi + +# REST JMX opts +if [ -n "$HBASE_JMX_OPTS" ] && [ -z "$HBASE_REST_JMX_OPTS" ]; then + HBASE_REST_JMX_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxremote.port=10105" +fi +# REST opts +if [ -z "$HBASE_REST_OPTS" ]; then + export HBASE_REST_OPTS="$HBASE_REST_JMX_OPTS" +fi + +# Source the hbase-env.sh. Will have JAVA_HOME defined. +# HBASE-7817 - Source the hbase-env.sh only if it has not already been done. HBASE_ENV_INIT keeps track of it. +if [ -z "$HBASE_ENV_INIT" ] && [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then + . "${HBASE_CONF_DIR}/hbase-env.sh" + export HBASE_ENV_INIT="true" +fi + +# Set default value for regionserver uid if not present +if [ -z "$HBASE_REGIONSERVER_UID" ]; then + HBASE_REGIONSERVER_UID="hbase" +fi + +# Verify if hbase has the mlock agent +if [ "$HBASE_REGIONSERVER_MLOCK" = "true" ]; then + MLOCK_AGENT="$HBASE_HOME/native/libmlockall_agent.so" + if [ ! -f "$MLOCK_AGENT" ]; then + cat 1>&2 <&2 < http://www.oracle.com/technetwork/java/javase/downloads | +| | +| HBase requires Java 1.7 or later. | ++======================================================================+ +EOF + exit 1 +fi diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh new file mode 100755 index 0000000000..4ad657f137 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemon.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Runs a Hadoop hbase command as a daemon. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_LOG_DIR Where log files are stored. PWD by default. +# HBASE_PID_DIR The pid files are stored. /tmp by default. +# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default +# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. +# HBASE_STOP_TIMEOUT Time, in seconds, after which we kill -9 the server if it has not stopped. +# Default 1200 seconds. +# +# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh + +usage="Usage: hbase-daemon.sh [--config ]\ + (start|stop|restart|autorestart) \ + " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh +. "$bin"/hbase-common.sh + +# get arguments +startStop=$1 +shift + +command=$1 +shift + +hbase_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" + num=$prev + done + mv -f "$log" "$log.$num"; + fi +} + +cleanZNode() { + if [ -f $HBASE_ZNODE_FILE ]; then + if [ "$command" = "master" ]; then + $bin/hbase master clear > /dev/null 2>&1 + else + #call ZK to delete the node + ZNODE=`cat $HBASE_ZNODE_FILE` + $bin/hbase zkcli delete $ZNODE > /dev/null 2>&1 + fi + rm $HBASE_ZNODE_FILE + fi +} + +check_before_start(){ + #ckeck if the process is not running + mkdir -p "$HBASE_PID_DIR" + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi +} + +wait_until_done () +{ + p=$1 + cnt=${HBASE_SLAVE_TIMEOUT:-300} + origcnt=$cnt + while kill -0 $p > /dev/null 2>&1; do + if [ $cnt -gt 1 ]; then + cnt=`expr $cnt - 1` + sleep 1 + else + echo "Process did not complete after $origcnt seconds, killing." + kill -9 $p + exit 1 + fi + done + return 0 +} + +# get log directory +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_PID_DIR" = "" ]; then + HBASE_PID_DIR=/tmp +fi + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +# Some variables +# Work out java location so can print version into log. +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log + +if [ -z "${HBASE_ROOT_LOGGER}" ]; then +export HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-"INFO,RFA"} +fi + +if [ -z "${HBASE_SECURITY_LOGGER}" ]; then +export HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-"INFO,RFAS"} +fi + +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid +export HBASE_ZNODE_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.znode +export HBASE_START_FILE=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.autorestart + +if [ -n "$SERVER_GC_OPTS" ]; then + export SERVER_GC_OPTS=${SERVER_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} +fi +if [ -n "$CLIENT_GC_OPTS" ]; then + export CLIENT_GC_OPTS=${CLIENT_GC_OPTS/"-Xloggc:"/"-Xloggc:${loggc}"} +fi + +# Set default scheduling priority +if [ "$HBASE_NICENESS" = "" ]; then + export HBASE_NICENESS=0 +fi + +thiscmd="$bin/$(basename ${BASH_SOURCE-$0})" +args=$@ + +case $startStop in + +(start) + check_before_start + hbase_rotate_log $logout + hbase_rotate_log $loggc + echo starting $command, logging to $logout + nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args < /dev/null > ${logout} 2>&1 & + sleep 1; head "${logout}" + ;; + +(autorestart) + check_before_start + hbase_rotate_log $logout + hbase_rotate_log $loggc + nohup $thiscmd --config "${HBASE_CONF_DIR}" internal_autorestart $command $args < /dev/null > ${logout} 2>&1 & + ;; + +(internal_start) + # Add to the command log file vital stats on our environment. + echo "`date` Starting $command on `hostname`" >> $loglog + echo "`ulimit -a`" >> $loglog 2>&1 + nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + $command "$@" start >> "$logout" 2>&1 & + echo $! > $pid + wait + cleanZNode + ;; + +(internal_autorestart) + touch "$HBASE_START_FILE" + #keep starting the command until asked to stop. Reloop on software crash + while true + do + lastLaunchDate=`date +%s` + $thiscmd --config "${HBASE_CONF_DIR}" internal_start $command $args + + #if the file does not exist it means that it was not stopped properly by the stop command + if [ ! -f "$HBASE_START_FILE" ]; then + exit 1 + fi + + #if the cluster is being stopped then do not restart it again. + zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` + if [ "$zparent" == "null" ]; then zparent="/hbase"; fi + zkrunning=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.state` + if [ "$zkrunning" == "null" ]; then zkrunning="running"; fi + zkFullRunning=$zparent/$zkrunning + $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep "Node does not exist" 1>/dev/null 2>&1 + #grep returns 0 if it found something, 1 otherwise + if [ $? -eq 0 ]; then + exit 1 + fi + + #If ZooKeeper cannot be found, then do not restart + $bin/hbase zkcli stat $zkFullRunning 2>&1 | grep Exception | grep ConnectionLoss 1>/dev/null 2>&1 + if [ $? -eq 0 ]; then + exit 1 + fi + + #if it was launched less than 5 minutes ago, then wait for 5 minutes before starting it again. + curDate=`date +%s` + limitDate=`expr $lastLaunchDate + 300` + if [ $limitDate -gt $curDate ]; then + sleep 300 + fi + done + ;; + +(stop) + rm -f "$HBASE_START_FILE" + if [ -f $pid ]; then + pidToKill=`cat $pid` + # kill -0 == see if the PID exists + if kill -0 $pidToKill > /dev/null 2>&1; then + echo -n stopping $command + echo "`date` Terminating $command" >> $loglog + kill $pidToKill > /dev/null 2>&1 + waitForProcessEnd $pidToKill $command + else + retval=$? + echo no $command to stop because kill -0 of pid $pidToKill failed with status $retval + fi + else + echo no $command to stop because no pid file $pid + fi + rm -f $pid + ;; + +(restart) + # stop the command + $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & + wait_until_done $! + # wait a user-specified sleep period + sp=${HBASE_RESTART_SLEEP:-3} + if [ $sp -gt 0 ]; then + sleep $sp + fi + # start the command + $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & + wait_until_done $! + ;; + +(*) + echo $usage + exit 1 + ;; +esac diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh new file mode 100755 index 0000000000..21ce635777 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-daemons.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a hbase command on all slave hosts. +# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh + +usage="Usage: hbase-daemons.sh [--config ] \ + [--hosts regionserversfile] [start|stop] command args..." + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. $bin/hbase-config.sh + +remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" +args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" + +command=$2 +case $command in + (zookeeper) + exec "$bin/zookeepers.sh" $args + ;; + (master-backup) + exec "$bin/master-backup.sh" $args + ;; + (*) + exec "$bin/regionservers.sh" $args + ;; +esac + diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby new file mode 100755 index 0000000000..37bce46f6b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase-jruby @@ -0,0 +1,22 @@ +#!/bin/bash +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + + +`dirname $0`/hbase org.jruby.Main $* + diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd new file mode 100755 index 0000000000..da7bcd9dea --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hbase.cmd @@ -0,0 +1,439 @@ +@echo off +@rem/* +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ +@rem +@rem The hbase command script. Based on the hadoop command script putting +@rem in hbase classes, libs and configurations ahead of hadoop's. +@rem +@rem TODO: Narrow the amount of duplicated code. +@rem +@rem Environment Variables: +@rem +@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +@rem +@rem HBASE_CLASSPATH Extra Java CLASSPATH entries. +@rem +@rem HBASE_HEAPSIZE The maximum amount of heap to use. +@rem Default is unset and uses the JVMs default setting +@rem (usually 1/4th of the available memory). +@rem +@rem HBASE_OPTS Extra Java runtime options. +@rem +@rem HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. +@rem +@rem HBASE_ROOT_LOGGER The root appender. Default is INFO,console +@rem +@rem JRUBY_HOME JRuby path: $JRUBY_HOME\lib\jruby.jar should exist. +@rem Defaults to the jar packaged with HBase. +@rem +@rem JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. +@rem Empty by default. +@rem HBASE_SHELL_OPTS Extra options passed to the hbase shell. +@rem Empty by default. + + +setlocal enabledelayedexpansion + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +rem This will set HBASE_HOME, etc. +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call "%hbase-config-script%" %* +if "%1" == "--config" ( + shift + shift +) + +rem Detect if we are in hbase sources dir +set in_dev_env=false + +if exist "%HBASE_HOME%\target" set in_dev_env=true + +rem --service is an internal option. used by MSI setup to install HBase as a windows service +if "%1" == "--service" ( + set service_entry=true + shift +) + +set hbase-command=%1 +shift + +@rem if no args specified, show usage +if "%hbase-command%"=="" ( + goto :print_usage + endlocal + goto :eof +) + +set JAVA_HEAP_MAX= +set JAVA_OFFHEAP_MAX= + +rem check envvars which might override default args +if defined HBASE_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%HBASE_HEAPSIZE%m +) + +if defined HBASE_OFFHEAPSIZE ( + set JAVA_OFFHEAP_MAX=-XX:MaxDirectMemory=%HBASE_OFFHEAPSIZE%m +) + +set CLASSPATH=%HBASE_CONF_DIR%;%JAVA_HOME%\lib\tools.jar + +rem Add maven target directory +set cached_classpath_filename=%HBASE_HOME%\target\cached_classpath.txt +if "%in_dev_env%"=="true" ( + + rem adding maven main classes to classpath + for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( + if exist %%i\target\classes set CLASSPATH=!CLASSPATH!;%%i\target\classes + ) + + rem adding maven test classes to classpath + rem For developers, add hbase classes to CLASSPATH + for /f %%i in ('dir /b "%HBASE_HOME%\hbase-*"') do ( + if exist %%i\target\test-classes set CLASSPATH=!CLASSPATH!;%%i\target\test-classes + ) + + if not exist "%cached_classpath_filename%" ( + echo "As this is a development environment, we need %cached_classpath_filename% to be generated from maven (command: mvn install -DskipTests)" + goto :eof + ) + + for /f "delims=" %%i in ('type "%cached_classpath_filename%"') do set CLASSPATH=%CLASSPATH%;%%i +) + +@rem For releases add hbase webapps to CLASSPATH +@rem Webapps must come first else it messes up Jetty +if exist "%HBASE_HOME%\hbase-webapps" ( + set CLASSPATH=%CLASSPATH%;%HBASE_HOME% +) + +if exist "%HBASE_HOME%\target\hbase-webapps" ( + set CLASSPATH=%CLASSPATH%;%HBASE_HOME%\target +) + +for /F %%f in ('dir /b "%HBASE_HOME%\hbase*.jar" 2^>nul') do ( + if not "%%f:~-11"=="sources.jar" ( + set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\%%f + ) +) + +@rem Add libs to CLASSPATH +if exist "%HBASE_HOME%\lib" ( + set CLASSPATH=!CLASSPATH!;%HBASE_HOME%\lib\* +) + +@rem Add user-specified CLASSPATH last +if defined HBASE_CLASSPATH ( + set CLASSPATH=%CLASSPATH%;%HBASE_CLASSPATH% +) + +@rem Default log directory and file +if not defined HBASE_LOG_DIR ( + set HBASE_LOG_DIR=%HBASE_HOME%\logs +) + +if not defined HBASE_LOGFILE ( + set HBASE_LOGFILE=hbase.log +) + +set JAVA_PLATFORM= + +rem If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +set PATH=%PATH%;"%HADOOP_HOME%\bin" +set HADOOP_IN_PATH=hadoop.cmd + +if exist "%HADOOP_HOME%\bin\%HADOOP_IN_PATH%" ( + set hadoopCpCommand=call %HADOOP_IN_PATH% classpath 2^>nul + for /f "eol= delims=" %%i in ('!hadoopCpCommand!') do set CLASSPATH_FROM_HADOOP=%%i + if defined CLASSPATH_FROM_HADOOP ( + set CLASSPATH=%CLASSPATH%;!CLASSPATH_FROM_HADOOP! + ) + set HADOOP_CLASSPATH=%CLASSPATH% + + set hadoopJLPCommand=call %HADOOP_IN_PATH% org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2^>nul + for /f "eol= delims=" %%i in ('!hadoopJLPCommand!') do set HADOOP_JAVA_LIBRARY_PATH=%%i + if not defined JAVA_LIBRARY_PATH ( + set JAVA_LIBRARY_PATH=!HADOOP_JAVA_LIBRARY_PATH! + ) else ( + set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!HADOOP_JAVA_LIBRARY_PATH! + ) +) + +if exist "%HBASE_HOME%\build\native" ( + set platformCommand=call %JAVA% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName + for /f %%i in ('!platformCommand!') do set JAVA_PLATFORM=%%i + set _PATH_TO_APPEND=%HBASE_HOME%\build\native\!JAVA_PLATFORM!;%HBASE_HOME%\build\native\!JAVA_PLATFORM!\lib + if not defined JAVA_LIBRARY_PATH ( + set JAVA_LIBRARY_PATH=!_PATH_TO_APPEND! + ) else ( + set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;!_PATH_TO_APPEND! + ) +) + +rem This loop would set %hbase-command-arguments% +set _hbasearguments= +:MakeCmdArgsLoop + if [%1]==[] goto :EndLoop + + if not defined _hbasearguments ( + set _hbasearguments=%1 + ) else ( + set _hbasearguments=!_hbasearguments! %1 + ) + shift +goto :MakeCmdArgsLoop +:EndLoop + +set hbase-command-arguments=%_hbasearguments% + +@rem figure out which class to run +set corecommands=shell master regionserver thrift thrift2 rest avro hlog wal hbck hfile zookeeper zkcli upgrade mapredcp +for %%i in ( %corecommands% ) do ( + if "%hbase-command%"=="%%i" set corecommand=true +) + +if defined corecommand ( + call :%hbase-command% %hbase-command-arguments% +) else ( + if "%hbase-command%" == "classpath" ( + echo %CLASSPATH% + goto :eof + ) + if "%hbase-command%" == "version" ( + set CLASS=org.apache.hadoop.hbase.util.VersionInfo + ) else ( + set CLASS=%hbase-command% + ) +) + +if not defined HBASE_IDENT_STRING ( + set HBASE_IDENT_STRING=%USERNAME% +) + +@rem Set the right GC options based on the what we are running +set servercommands=master regionserver thrift thrift2 rest avro zookeeper +for %%i in ( %servercommands% ) do ( + if "%hbase-command%"=="%%i" set servercommand=true +) + +if "%servercommand%" == "true" ( + set HBASE_OPTS=%HBASE_OPTS% %SERVER_GC_OPTS% +) else ( + set HBASE_OPTS=%HBASE_OPTS% %CLIENT_GC_OPTS% +) + +@rem If HBase is run as a windows service, configure logging +if defined service_entry ( + set HBASE_LOG_PREFIX=hbase-%hbase-command%-%COMPUTERNAME% + set HBASE_LOGFILE=!HBASE_LOG_PREFIX!.log + if not defined HBASE_ROOT_LOGGER ( + set HBASE_ROOT_LOGGER=INFO,DRFA + ) + set HBASE_SECURITY_LOGGER=INFO,DRFAS + set loggc=!HBASE_LOG_DIR!\!HBASE_LOG_PREFIX!.gc + set loglog=!HBASE_LOG_DIR!\!HBASE_LOGFILE! + + if "%HBASE_USE_GC_LOGFILE%" == "true" ( + set HBASE_OPTS=%HBASE_OPTS% -Xloggc:"!loggc!" + ) +) + + +@rem Have JVM dump heap if we run out of memory. Files will be 'launch directory' +@rem and are named like the following: java_pid21612.hprof. Apparently it does not +@rem 'cost' to have this flag enabled. Its a 1.6 flag only. See: +@rem http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.dir="%HBASE_LOG_DIR%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.log.file="%HBASE_LOGFILE%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.home.dir="%HBASE_HOME%" +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.id.str="%HBASE_IDENT_STRING%" +set HBASE_OPTS=%HBASE_OPTS% -XX:OnOutOfMemoryError="taskkill /F /PID %p" + +if not defined HBASE_ROOT_LOGGER ( + set HBASE_ROOT_LOGGER=INFO,console +) +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger="%HBASE_ROOT_LOGGER%" + +if defined JAVA_LIBRARY_PATH ( + set HBASE_OPTS=%HBASE_OPTS% -Djava.library.path="%JAVA_LIBRARY_PATH%" +) + +rem Enable security logging on the master and regionserver only +if not defined HBASE_SECURITY_LOGGER ( + set HBASE_SECURITY_LOGGER=INFO,NullAppender + if "%hbase-command%"=="master" ( + set HBASE_SECURITY_LOGGER=INFO,DRFAS + ) + if "%hbase-command%"=="regionserver" ( + set HBASE_SECURITY_LOGGER=INFO,DRFAS + ) +) +set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%" + +set HEAP_SETTINGS=%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX% +set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments% + +if defined service_entry ( + call :makeServiceXml %java_arguments% +) else ( + call %JAVA% %java_arguments% +) + +endlocal +goto :eof + +:shell + rem eg export JRUBY_HOME=/usr/local/share/jruby + if defined JRUBY_HOME ( + set CLASSPATH=%CLASSPATH%;%JRUBY_HOME%\lib\jruby.jar + set HBASE_OPTS=%HBASE_OPTS% -Djruby.home="%JRUBY_HOME%" -Djruby.lib="%JRUBY_HOME%\lib" + ) + rem find the hbase ruby sources + if exist "%HBASE_HOME%\lib\ruby" ( + set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\lib\ruby" + ) else ( + set HBASE_OPTS=%HBASE_OPTS% -Dhbase.ruby.sources="%HBASE_HOME%\hbase-shell\src\main\ruby" + ) + set HBASE_OPTS=%HBASE_OPTS% %HBASE_SHELL_OPTS% + + set CLASS=org.jruby.Main -X+O %JRUBY_OPTS% "%HBASE_HOME%\bin\hirb.rb" + goto :eof + +:master + set CLASS=org.apache.hadoop.hbase.master.HMaster + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_MASTER_OPTS% + ) + goto :eof + +:regionserver + set CLASS=org.apache.hadoop.hbase.regionserver.HRegionServer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_REGIONSERVER_OPTS% + ) + goto :eof + +:thrift + set CLASS=org.apache.hadoop.hbase.thrift.ThriftServer + if NOT "%1" == "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% + ) + goto :eof + +:thrift2 + set CLASS=org.apache.hadoop.hbase.thrift2.ThriftServer + if NOT "%1" == "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_THRIFT_OPTS% + ) + goto :eof + +:rest + set CLASS=org.apache.hadoop.hbase.rest.RESTServer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_REST_OPTS% + ) + goto :eof + +:avro + set CLASS=org.apache.hadoop.hbase.avro.AvroServer + if NOT "%1"== "stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_AVRO_OPTS% + ) + goto :eof + +:zookeeper + set CLASS=org.apache.hadoop.hbase.zookeeper.HQuorumPeer + if NOT "%1"=="stop" ( + set HBASE_OPTS=%HBASE_OPTS% %HBASE_ZOOKEEPER_OPTS% + ) + goto :eof + +:hbck + set CLASS=org.apache.hadoop.hbase.util.HBaseFsck + goto :eof + +@rem TODO remove older 'hlog' command +:hlog + set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter + goto :eof + +:wal + set CLASS=org.apache.hadoop.hbase.wal.WALPrettyPrinter + goto :eof + +:hfile + set CLASS=org.apache.hadoop.hbase.io.hfile.HFile + goto :eof + +:zkcli + set CLASS=org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer + goto :eof + +:upgrade + set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96 + goto :eof + +:mapredcp + set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool + goto :eof + +:makeServiceXml + set arguments=%* + @echo ^ + @echo ^%hbase-command%^ + @echo ^%hbase-command%^ + @echo ^This service runs Isotope %hbase-command%^ + @echo ^%JAVA%^ + @echo ^%arguments%^ + @echo ^ + goto :eof + +:print_usage + echo Usage: hbase [^] ^ [^] + echo where ^ an option from one of these categories:: + echo Options: + echo --config DIR Configuration direction to use. Default: ./conf + echo. + echo Commands: + echo Some commands take arguments. Pass no args or -h for usage." + echo shell Run the HBase shell + echo hbck Run the hbase 'fsck' tool + echo wal Write-ahead-log analyzer + echo hfile Store file analyzer + echo zkcli Run the ZooKeeper shell + echo upgrade Upgrade hbase + echo master Run an HBase HMaster node + echo regionserver Run an HBase HRegionServer node + echo zookeeper Run a Zookeeper server + echo rest Run an HBase REST server + echo thrift Run the HBase Thrift server + echo thrift2 Run the HBase Thrift2 server + echo classpath Dump hbase CLASSPATH + echo mapredcp Dump CLASSPATH entries required by mapreduce + echo version Print the version + echo CLASSNAME Run the class named CLASSNAME + goto :eof diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb new file mode 100755 index 0000000000..0503c29706 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/hirb.rb @@ -0,0 +1,195 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. + +# TODO: Interrupt a table creation or a connection to a bad master. Currently +# has to time out. Below we've set down the retries for rpc and hbase but +# still can be annoying (And there seem to be times when we'll retry for +# ever regardless) +# TODO: Add support for listing and manipulating catalog tables, etc. +# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes + +# Run the java magic include and import basic HBase types that will help ease +# hbase hacking. +include Java + +# Some goodies for hirb. Should these be left up to the user's discretion? +require 'irb/completion' +require 'pathname' + +# Add the directory names in hbase.jruby.sources commandline option +# to the ruby load path so I can load up my HBase ruby modules +sources = java.lang.System.getProperty('hbase.ruby.sources') +$LOAD_PATH.unshift Pathname.new(sources) + +# +# FIXME: Switch args processing to getopt +# +# See if there are args for this shell. If any, read and then strip from ARGV +# so they don't go through to irb. Output shell 'usage' if user types '--help' +cmdline_help = </dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [--config ] [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1" + exit +fi + +. "$bin"/hbase-config.sh + +# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] +export HBASE_MASTER_OPTS=" " + +run_master () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_MASTER_ARGS="\ + -D hbase.master.info.port=`expr 16010 + $DN` \ + -D hbase.regionserver.port=`expr 16020 + $DN` \ + -D hbase.regionserver.info.port=`expr 16030 + $DN` \ + --backup" + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 master $HBASE_MASTER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + if [[ "$i" =~ ^[0-9]+$ ]]; then + run_master $cmd $i + else + echo "Invalid argument" + fi +done diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh new file mode 100755 index 0000000000..78b4195811 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/local-regionservers.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# This is used for starting multiple regionservers on the same machine. +# run it from hbase-dir/ just like 'bin/hbase' +# Supports up to 100 regionservers (limitation = overlapping ports) + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin" >/dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [--config ] [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1 2" + exit +fi + +. "$bin"/hbase-config.sh + +# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] +export HBASE_REGIONSERVER_OPTS=" " + +run_regionserver () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_REGIONSERVER_ARGS="\ + -D hbase.regionserver.port=`expr 16200 + $DN` \ + -D hbase.regionserver.info.port=`expr 16300 + $DN`" + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $1 regionserver $HBASE_REGIONSERVER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + if [[ "$i" =~ ^[0-9]+$ ]]; then + run_regionserver $cmd $i + else + echo "Invalid argument" + fi +done diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh new file mode 100755 index 0000000000..feca4ab865 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/master-backup.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all backup master hosts. +# +# Environment Variables +# +# HBASE_BACKUP_MASTERS File naming remote hosts. +# Default is ${HBASE_CONF_DIR}/backup-masters +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: $0 [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the master backup file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_BACKUP_MASTERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_BACKUP_MASTERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" + else + export HOSTLIST="${HBASE_BACKUP_MASTERS}" + fi +fi + + +args=${@// /\\ } +args=${args/master-backup/master} + +if [ -f $HOSTLIST ]; then + for hmaster in `cat "$HOSTLIST"`; do + ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ + 2>&1 | sed "s/^/$hmaster: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb new file mode 100755 index 0000000000..134b032b87 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_mover.rb @@ -0,0 +1,488 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Moves regions. Will confirm region access in current location and will +# not move a new region until successful confirm of region loading in new +# location. Presumes balancer is disabled when we run (not harmful if its +# on but this script and balancer will end up fighting each other). +require 'optparse' +require File.join(File.dirname(__FILE__), 'thread-pool') +include Java +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.client.Get +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.client.HTable +import org.apache.hadoop.hbase.client.HConnectionManager +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.InclusiveStopFilter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.util.Writables +import org.apache.hadoop.conf.Configuration +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory +import org.apache.hadoop.hbase.protobuf.ProtobufUtil +import org.apache.hadoop.hbase.ServerName +import org.apache.hadoop.hbase.HRegionInfo + +# Name of this script +NAME = "region_mover" + +# Returns true if passed region is still on 'original' when we look at hbase:meta. +def isSameServer(admin, r, original) + server = getServerNameForRegion(admin, r) + return false unless server and original + return server == original +end + +class RubyAbortable + include org.apache.hadoop.hbase.Abortable + def abort(why, e) + puts "ABORTED! why=" + why + ", e=" + e.to_s + end +end + +# Get servername that is up in hbase:meta; this is hostname + port + startcode comma-delimited. +# Can return nil +def getServerNameForRegion(admin, r) + return nil unless admin.isTableEnabled(r.getTableName) + if r.isMetaRegion() + # Hack + zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(admin.getConfiguration(), "region_mover", nil) + mtl = org.apache.hadoop.hbase.zookeeper.MetaTableLocator.new() + begin + while not mtl.isLocationAvailable(zkw) + sleep 0.1 + end + # Make a fake servername by appending ',' + metaServer = mtl.getMetaRegionLocation(zkw).toString() + "," + return metaServer + ensure + zkw.close() + end + end + table = HTable.new(admin.getConfiguration(), HConstants::META_TABLE_NAME) + begin + g = Get.new(r.getRegionName()) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + g.addColumn(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + result = table.get(g) + return nil unless result + server = result.getValue(HConstants::CATALOG_FAMILY, HConstants::SERVER_QUALIFIER) + startcode = result.getValue(HConstants::CATALOG_FAMILY, HConstants::STARTCODE_QUALIFIER) + return nil unless server + return java.lang.String.new(Bytes.toString(server)).replaceFirst(":", ",") + "," + Bytes.toLong(startcode).to_s + ensure + table.close() + end +end + +# Trys to scan a row from passed region +# Throws exception if can't +def isSuccessfulScan(admin, r) + scan = Scan.new(r.getStartKey(), r.getStartKey()) + scan.setBatch(1) + scan.setCaching(1) + scan.setFilter(FilterList.new(FirstKeyOnlyFilter.new(),InclusiveStopFilter.new(r.getStartKey()))) + begin + table = HTable.new(admin.getConfiguration(), r.getTableName()) + scanner = table.getScanner(scan) + begin + results = scanner.next() + # We might scan into next region, this might be an empty table. + # But if no exception, presume scanning is working. + ensure + scanner.close() + end + rescue org.apache.hadoop.hbase.TableNotFoundException, + org.apache.hadoop.hbase.TableNotEnabledException => e + $LOG.warn("Region " + r.getEncodedName() + " belongs to recently " + + "deleted/disabled table. Skipping... " + e.message) + return + ensure + table.close() unless table.nil? + end +end + +# Check region has moved successful and is indeed hosted on another server +# Wait until that is the case. +def move(admin, r, newServer, original) + # Now move it. Do it in a loop so can retry if fail. Have seen issue where + # we tried move region but failed and retry put it back on old location; + # retry in this case. + + retries = admin.getConfiguration.getInt("hbase.move.retries.max", 5) + count = 0 + same = true + start = Time.now + while count < retries and same + if count > 0 + $LOG.info("Retry " + count.to_s + " of maximum " + retries.to_s) + end + count = count + 1 + begin + admin.move(Bytes.toBytes(r.getEncodedName()), Bytes.toBytes(newServer)) + rescue java.lang.reflect.UndeclaredThrowableException, + org.apache.hadoop.hbase.UnknownRegionException => e + $LOG.info("Exception moving " + r.getEncodedName() + + "; split/moved? Continuing: " + e) + return + end + # Wait till its up on new server before moving on + maxWaitInSeconds = admin.getConfiguration.getInt("hbase.move.wait.max", 60) + maxWait = Time.now + maxWaitInSeconds + while Time.now < maxWait + same = isSameServer(admin, r, original) + break unless same + sleep 0.1 + end + end + raise RuntimeError, "Region stuck on #{original}, newserver=#{newServer}" if same + # Assert can Scan from new location. + isSuccessfulScan(admin, r) + $LOG.info("Moved region " + r.getRegionNameAsString() + " cost: " + + java.lang.String.format("%.3f", (Time.now - start))) +end + +# Return the hostname:port out of a servername (all up to second ',') +def getHostPortFromServerName(serverName) + return serverName.split(',')[0..1] +end + +# Return array of servernames where servername is hostname+port+startcode +# comma-delimited +def getServers(admin) + serverInfos = admin.getClusterStatus().getServerInfo() + servers = [] + for server in serverInfos + servers << server.getServerName() + end + return servers +end + +# Remove the servername whose hostname portion matches from the passed +# array of servers. Returns as side-effect the servername removed. +def stripServer(servers, hostname, port) + count = servers.length + servername = nil + for server in servers + hostFromServerName, portFromServerName = getHostPortFromServerName(server) + if hostFromServerName == hostname and portFromServerName == port + servername = servers.delete(server) + end + end + # Check server to exclude is actually present + raise RuntimeError, "Server %s:%d not online" % [hostname, port] unless servers.length < count + return servername +end + +# Returns a new serverlist that excludes the servername whose hostname portion +# matches from the passed array of servers. +def stripExcludes(servers, excludefile) + excludes = readExcludes(excludefile) + servers = servers.find_all{|server| + !excludes.contains(getHostPortFromServerName(server).join(":")) + } + # return updated servers list + return servers +end + + +# Return servername that matches passed hostname and port +def getServerName(servers, hostname, port) + servername = nil + for server in servers + hostFromServerName, portFromServerName = getHostPortFromServerName(server) + if hostFromServerName == hostname and portFromServerName == port + servername = server + break + end + end + raise ArgumentError, "Server %s:%d not online" % [hostname, port] unless servername + return servername +end + +# Create a logger and disable the DEBUG-level annoying client logging +def configureLogging(options) + apacheLogger = LogFactory.getLog(NAME) + # Configure log4j to not spew so much + unless (options[:debug]) + logger = org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase.client") + logger.setLevel(org.apache.log4j.Level::INFO) + end + return apacheLogger +end + +# Get configuration instance +def getConfiguration() + config = HBaseConfiguration.create() + # No prefetching on hbase:meta This is for versions pre 0.99. Newer versions do not prefetch. + config.setInt("hbase.client.prefetch.limit", 1) + # Make a config that retries at short intervals many times + config.setInt("hbase.client.pause", 500) + config.setInt("hbase.client.retries.number", 100) + return config +end + +# Now get list of regions on targetServer +def getRegions(config, servername) + connection = HConnectionManager::getConnection(config); + return ProtobufUtil::getOnlineRegions(connection.getAdmin(ServerName.valueOf(servername))); +end + +def deleteFile(filename) + f = java.io.File.new(filename) + f.delete() if f.exists() +end + +# Write HRegionInfo to file +# Need to serialize in case non-printable characters. +# Format is count of regionnames followed by serialized regionnames. +def writeFile(filename, regions) + fos = java.io.FileOutputStream.new(filename) + dos = java.io.DataOutputStream.new(fos) + # Write out a count of region names + dos.writeInt(regions.size()) + # Write actual region names. + for r in regions + Bytes.writeByteArray(dos, r.toByteArray()) + end + dos.close() +end + +# See writeFile above. +# Returns array of HRegionInfos +def readFile(filename) + f = java.io.File.new(filename) + return java.util.ArrayList.new() unless f.exists() + fis = java.io.FileInputStream.new(f) + dis = java.io.DataInputStream.new(fis) + # Read count of regions + count = dis.readInt() + regions = java.util.ArrayList.new(count) + index = 0 + while index < count + regions.add(HRegionInfo.parseFromOrNull(Bytes.readByteArray(dis))) + index = index + 1 + end + dis.close() + return regions +end + +# Move regions off the passed hostname:port +def unloadRegions(options, hostname, port) + # Clean up any old files. + filename = getFilename(options, hostname, port) + deleteFile(filename) + # Get configuration + config = getConfiguration() + # Get an admin instance + admin = HBaseAdmin.new(config) + servers = getServers(admin) + # Remove the server we are unloading from from list of servers. + # Side-effect is the servername that matches this hostname + servername = stripServer(servers, hostname, port) + + # Remove the servers in our exclude list from list of servers. + servers = stripExcludes(servers, options[:excludesFile]) + puts "Valid region move targets: ", servers + if servers.length == 0 + puts "No regions were moved - there was no server available" + exit 4 + end + movedRegions = java.util.Collections.synchronizedList(java.util.ArrayList.new()) + while true + rs = getRegions(config, servername) + # Remove those already tried to move + rs.removeAll(movedRegions) + break if rs.length == 0 + $LOG.info("Moving " + rs.length.to_s + " region(s) from " + servername + + " on " + servers.length.to_s + " servers using " + options[:maxthreads].to_s + " threads.") + counter = 0 + pool = ThreadPool.new(options[:maxthreads]) + server_index = 0 + while counter < rs.length do + pool.launch(rs,counter,server_index) do |_rs,_counter,_server_index| + $LOG.info("Moving region " + _rs[_counter].getEncodedName() + " (" + (_counter + 1).to_s + + " of " + _rs.length.to_s + ") to server=" + servers[_server_index] + " for " + servername) + # Assert we can scan region in its current location + isSuccessfulScan(admin, _rs[_counter]) + # Now move it. + move(admin, _rs[_counter], servers[_server_index], servername) + movedRegions.add(_rs[_counter]) + end + counter += 1 + server_index = (server_index + 1) % servers.length + end + $LOG.info("Waiting for the pool to complete") + pool.stop + $LOG.info("Pool completed") + end + if movedRegions.size() > 0 + # Write out file of regions moved + writeFile(filename, movedRegions) + $LOG.info("Wrote list of moved regions to " + filename) + end +end + +# Move regions to the passed hostname +def loadRegions(options, hostname, port) + # Get configuration + config = getConfiguration() + # Get an admin instance + admin = HBaseAdmin.new(config) + filename = getFilename(options, hostname, port) + regions = readFile(filename) + return if regions.isEmpty() + servername = nil + # Wait till server is up + maxWaitInSeconds = admin.getConfiguration.getInt("hbase.serverstart.wait.max", 180) + maxWait = Time.now + maxWaitInSeconds + while Time.now < maxWait + servers = getServers(admin) + begin + servername = getServerName(servers, hostname, port) + rescue ArgumentError => e + $LOG.info("hostname=" + hostname.to_s + ":" + port.to_s + " is not up yet, waiting"); + end + break if servername + sleep 0.5 + end + $LOG.info("Moving " + regions.size().to_s + " regions to " + servername) + # sleep 20s to make sure the rs finished initialization. + sleep 20 + counter = 0 + pool = ThreadPool.new(options[:maxthreads]) + while counter < regions.length do + r = regions[counter] + exists = false + begin + isSuccessfulScan(admin, r) + exists = true + rescue org.apache.hadoop.hbase.NotServingRegionException => e + $LOG.info("Failed scan of " + e.message) + end + next unless exists + currentServer = getServerNameForRegion(admin, r) + if currentServer and currentServer == servername + $LOG.info("Region " + r.getRegionNameAsString() + " (" + counter.to_s + + " of " + regions.length.to_s + ") already on target server=" + servername) + counter = counter + 1 + next + end + pool.launch(r,currentServer,counter) do |_r,_currentServer,_counter| + $LOG.info("Moving region " + _r.getRegionNameAsString() + " (" + (_counter + 1).to_s + + " of " + regions.length.to_s + ") from " + _currentServer.to_s + " to server=" + + servername); + move(admin, _r, servername, _currentServer) + end + counter = counter + 1 + end + pool.stop +end + +# Returns an array of hosts to exclude as region move targets +def readExcludes(filename) + if filename == nil + return java.util.ArrayList.new() + end + if ! File.exist?(filename) + puts "Error: Unable to read host exclude file: ", filename + raise RuntimeError + end + + f = File.new(filename, "r") + # Read excluded hosts list + excludes = java.util.ArrayList.new() + while (line = f.gets) + line.strip! # do an inplace drop of pre and post whitespaces + excludes.add(line) unless line.empty? # exclude empty lines + end + puts "Excluding hosts as region move targets: ", excludes + f.close + + return excludes +end + +def getFilename(options, targetServer, port) + filename = options[:file] + if not filename + filename = "/tmp/" + ENV['USER'] + targetServer + ":" + port + end + return filename +end + + +# Do command-line parsing +options = {} +optparse = OptionParser.new do |opts| + opts.banner = "Usage: #{NAME}.rb [options] load|unload [|]" + opts.separator 'Load or unload regions by moving one at a time' + options[:file] = nil + options[:maxthreads] = 1 + opts.on('-f', '--filename=FILE', 'File to save regions list into unloading, or read from loading; default /tmp/') do |file| + options[:file] = file + end + opts.on('-h', '--help', 'Display usage information') do + puts opts + exit + end + options[:debug] = false + opts.on('-d', '--debug', 'Display extra debug logging') do + options[:debug] = true + end + opts.on('-x', '--excludefile=FILE', 'File with hosts-per-line to exclude as unload targets; default excludes only target host; useful for rack decommisioning.') do |file| + options[:excludesFile] = file + end + opts.on('-m', '--maxthreads=XX', 'Define the maximum number of threads to use to unload and reload the regions') do |number| + options[:maxthreads] = number.to_i + end +end +optparse.parse! + +# Check ARGVs +if ARGV.length < 2 + puts optparse + exit 1 +end +hostname, port = ARGV[1].split(":") +if not hostname + opts optparse + exit 2 +end + +# Get configuration +config = getConfiguration() +if not port + port = config.getInt(HConstants::REGIONSERVER_PORT, HConstants::DEFAULT_REGIONSERVER_PORT) +end +port = port.to_s + +# Create a logger and save it to ruby global +$LOG = configureLogging(options) +case ARGV[0] + when 'load' + loadRegions(options, hostname, port) + when 'unload' + unloadRegions(options, hostname, port) + else + puts optparse + exit 3 +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb new file mode 100755 index 0000000000..878d58ab5d --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/region_status.rb @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# View the current status of all regions on an HBase cluster. This is +# predominantly used to determined if all the regions in META have been +# onlined yet on startup. +# +# To use this script, run: +# +# ${HBASE_HOME}/bin/hbase org.jruby.Main region_status.rb [wait] [--table ] + + +require 'optparse' + +usage = 'Usage : ./hbase org.jruby.Main region_status.rb [wait]' + + '[--table ]\n' +OptionParser.new do |o| + o.banner = usage + o.on('-t', '--table TABLENAME', 'Only process TABLENAME') do |tablename| + $tablename = tablename + end + o.on('-h', '--help', 'Display help message') { puts o; exit } + o.parse! +end + +SHOULD_WAIT = ARGV[0] == 'wait' +if ARGV[0] and not SHOULD_WAIT + print usage + exit 1 +end + + +require 'java' + +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.MasterNotRunningException +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.client.HTable +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.HRegionInfo +import org.apache.hadoop.hbase.client.MetaScanner +import org.apache.hadoop.hbase.HTableDescriptor +import org.apache.hadoop.hbase.client.HConnectionManager + +# disable debug logging on this script for clarity +log_level = org.apache.log4j.Level::ERROR +org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level) +org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level) + +config = HBaseConfiguration.create +config.set 'fs.defaultFS', config.get(HConstants::HBASE_DIR) + +# wait until the master is running +admin = nil +while true + begin + admin = HBaseAdmin.new config + break + rescue MasterNotRunningException => e + print 'Waiting for master to start...\n' + sleep 1 + end +end + +meta_count = 0 +server_count = 0 + +# scan META to see how many regions we should have +if $tablename.nil? + scan = Scan.new +else + tableNameMetaPrefix = $tablename + HConstants::META_ROW_DELIMITER.chr + scan = Scan.new( + (tableNameMetaPrefix + HConstants::META_ROW_DELIMITER.chr).to_java_bytes + ) +end +scan.setCacheBlocks(false) +scan.setCaching(10) +scan.setFilter(FirstKeyOnlyFilter.new) +INFO = 'info'.to_java_bytes +REGION_INFO = 'regioninfo'.to_java_bytes +scan.addColumn INFO, REGION_INFO +table = nil +iter = nil +while true + begin + table = HTable.new config, 'hbase:meta'.to_java_bytes + scanner = table.getScanner(scan) + iter = scanner.iterator + break + rescue IOException => ioe + print "Exception trying to scan META: #{ioe}" + sleep 1 + end +end +while iter.hasNext + result = iter.next + rowid = Bytes.toString(result.getRow()) + rowidStr = java.lang.String.new(rowid) + if not $tablename.nil? and not rowidStr.startsWith(tableNameMetaPrefix) + # Gone too far, break + break + end + region = HRegionInfo.getHRegionInfo result + if not region.isOffline + # only include regions that should be online + meta_count += 1 + end +end +scanner.close +# If we're trying to see the status of all HBase tables, we need to include the +# hbase:meta table, that is not included in our scan +if $tablename.nil? + meta_count += 1 +end + +# query the master to see how many regions are on region servers +if not $tablename.nil? + $TableName = HTableDescriptor.new($tablename.to_java_bytes).getTableName() +end +while true + if $tablename.nil? + server_count = admin.getClusterStatus().getRegionsCount() + else + connection = HConnectionManager::getConnection(config); + server_count = MetaScanner::allTableRegions(config, connection, $TableName ,false).size() + end + print "Region Status: #{server_count} / #{meta_count}\n" + if SHOULD_WAIT and server_count < meta_count + #continue this loop until server & meta count match + sleep 10 + else + break + end +end + +exit server_count == meta_count ? 0 : 1 diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh new file mode 100755 index 0000000000..af26ba95d7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/regionservers.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: regionservers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the regionservers file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_REGIONSERVERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_REGIONSERVERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/regionservers" + else + export HOSTLIST="${HBASE_REGIONSERVERS}" + fi +fi + +regionservers=`cat "$HOSTLIST"` +if [ "$regionservers" = "localhost" ]; then + "$bin"/local-regionservers.sh start 1 +else + for regionserver in `cat "$HOSTLIST"`; do + if ${HBASE_SLAVE_PARALLEL:-true}; then + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" & + else # run each command serially + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" + fi + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb new file mode 100755 index 0000000000..bc70031cb9 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/replication/copy_tables_desc.rb @@ -0,0 +1,73 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script to recreate all tables from one cluster to another +# To see usage for this script, run: +# +# ${HBASE_HOME}/bin/hbase org.jruby.Main copy_tables_desc.rb +# + +include Java +import org.apache.commons.logging.LogFactory +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.EmptyWatcher +import org.apache.hadoop.hbase.client.HBaseAdmin +import org.apache.hadoop.hbase.HTableDescriptor +import org.apache.hadoop.conf.Configuration + +# Name of this script +NAME = "copy_tables_desc" + +# Print usage for this script +def usage + puts 'Usage: %s.rb master_zookeeper.quorum.peers:clientport:znode_parent slave_zookeeper.quorum.peers:clientport:znode_parent' % NAME + exit! +end + +if ARGV.size != 2 + usage +end + +LOG = LogFactory.getLog(NAME) + +parts1 = ARGV[0].split(":") + +parts2 = ARGV[1].split(":") + +c1 = HBaseConfiguration.create() +c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0]) +c1.set("hbase.zookeeper.property.clientPort", parts1[1]) +c1.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts1[2]) + +admin1 = HBaseAdmin.new(c1) + +c2 = HBaseConfiguration.create() +c2.set(HConstants::ZOOKEEPER_QUORUM, parts2[0]) +c2.set("hbase.zookeeper.property.clientPort", parts2[1]) +c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2]) + +admin2 = HBaseAdmin.new(c2) + +for t in admin1.listTables() + admin2.createTable(t) +end + + +puts "All descriptions were copied" diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh new file mode 100755 index 0000000000..b3ac67f050 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/rolling-restart.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage_str="Usage: `basename $0` [--config ] [--rs-only] [--master-only] [--graceful] [--maxthreads xx]" + +function usage() { + echo "${usage_str}" +} + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi + +RR_RS=1 +RR_MASTER=1 +RR_GRACEFUL=0 +RR_MAXTHREADS=1 + +while [ $# -gt 0 ]; do + case "$1" in + --rs-only|-r) + RR_RS=1 + RR_MASTER=0 + RR_GRACEFUL=0 + shift + ;; + --master-only) + RR_RS=0 + RR_MASTER=1 + RR_GRACEFUL=0 + shift + ;; + --graceful) + RR_RS=0 + RR_MASTER=0 + RR_GRACEFUL=1 + shift + ;; + --maxthreads) + shift + RR_MAXTHREADS=$1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo Bad argument: $1 + usage + exit 1 + ;; + esac +done + +# quick function to get a value from the HBase config file +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`HBASE_CONF_DIR=${HBASE_CONF_DIR} $bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'false' ]; then + if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then + echo Cant do selective rolling restart if not running distributed + exit 1 + fi + "$bin"/hbase-daemon.sh restart master +else + zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` + if [ "$zparent" == "null" ]; then zparent="/hbase"; fi + + if [ $RR_MASTER -eq 1 ]; then + # stop all masters before re-start to avoid races for master znode + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup + + # make sure the master znode has been deleted before continuing + zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` + if [ "$zmaster" == "null" ]; then zmaster="master"; fi + zmaster=$zparent/$zmaster + echo -n "Waiting for Master ZNode ${zmaster} to expire" + while ! "$bin"/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do + echo -n "." + sleep 1 + done + echo #force a newline + + # all masters are down, now restart + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" start master-backup + + echo "Wait a minute for master to come up join cluster" + sleep 60 + + # Master joing cluster will start in cleaning out regions in transition. + # Wait until the master has cleaned out regions in transition before + # giving it a bunch of work to do; master is vulnerable during startup + zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` + if [ "$zunassigned" == "null" ]; then zunassigned="region-in-transition"; fi + zunassigned="$zparent/$zunassigned" + echo -n "Waiting for ${zunassigned} to empty" + while true ; do + unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` + if test 0 -eq ${unassigned} + then + break + else + echo -n " ${unassigned}" + fi + sleep 1 + done + fi + + if [ $RR_RS -eq 1 ]; then + # unlike the masters, roll all regionservers one-at-a-time + export HBASE_SLAVE_PARALLEL=false + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" restart regionserver + fi + + if [ $RR_GRACEFUL -eq 1 ]; then + # gracefully restart all online regionservers + zkrs=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.rs` + if [ "$zkrs" == "null" ]; then zkrs="rs"; fi + zkrs="$zparent/$zkrs" + online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` + for rs in $online_regionservers + do + rs_parts=(${rs//,/ }) + hostname=${rs_parts[0]} + echo "Gracefully restarting: $hostname" + "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug --maxthreads "${RR_MAXTHREADS}" "$hostname" + sleep 1 + done + fi +fi diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb new file mode 100644 index 0000000000..304afaf014 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase.rb @@ -0,0 +1,94 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# HBase ruby classes. +# Has wrapper classes for org.apache.hadoop.hbase.client.Admin +# and for org.apache.hadoop.hbase.client.Table. Classes take +# Formatters on construction and outputs any results using +# Formatter methods. These classes are only really for use by +# the hirb.rb HBase Shell script; they don't make much sense elsewhere. +# For example, the exists method on Admin class prints to the formatter +# whether the table exists and returns nil regardless. +include Java + +include_class('java.lang.Integer') {|package,name| "J#{name}" } +include_class('java.lang.Long') {|package,name| "J#{name}" } +include_class('java.lang.Boolean') {|package,name| "J#{name}" } + +module HBaseConstants + COLUMN = "COLUMN" + COLUMNS = "COLUMNS" + TIMESTAMP = "TIMESTAMP" + TIMERANGE = "TIMERANGE" + NAME = org.apache.hadoop.hbase.HConstants::NAME + VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS + IN_MEMORY = org.apache.hadoop.hbase.HConstants::IN_MEMORY + METADATA = org.apache.hadoop.hbase.HConstants::METADATA + STOPROW = "STOPROW" + STARTROW = "STARTROW" + ENDROW = STOPROW + RAW = "RAW" + LIMIT = "LIMIT" + METHOD = "METHOD" + MAXLENGTH = "MAXLENGTH" + CACHE_BLOCKS = "CACHE_BLOCKS" + REVERSED = "REVERSED" + REPLICATION_SCOPE = "REPLICATION_SCOPE" + INTERVAL = 'INTERVAL' + CACHE = 'CACHE' + FILTER = 'FILTER' + SPLITS = 'SPLITS' + SPLITS_FILE = 'SPLITS_FILE' + SPLITALGO = 'SPLITALGO' + NUMREGIONS = 'NUMREGIONS' + REGION_REPLICATION = 'REGION_REPLICATION' + REGION_REPLICA_ID = 'REGION_REPLICA_ID' + CONFIGURATION = org.apache.hadoop.hbase.HConstants::CONFIGURATION + ATTRIBUTES="ATTRIBUTES" + VISIBILITY="VISIBILITY" + AUTHORIZATIONS = "AUTHORIZATIONS" + SKIP_FLUSH = 'SKIP_FLUSH' + CONSISTENCY = "CONSISTENCY" + ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME' + CLUSTER_KEY = 'CLUSTER_KEY' + TABLE_CFS = 'TABLE_CFS' + CONFIG = 'CONFIG' + DATA = 'DATA' + + # Load constants from hbase java API + def self.promote_constants(constants) + # The constants to import are all in uppercase + constants.each do |c| + next if c =~ /DEFAULT_.*/ || c != c.upcase + next if eval("defined?(#{c})") + eval("#{c} = '#{c}'") + end + end + + promote_constants(org.apache.hadoop.hbase.HColumnDescriptor.constants) + promote_constants(org.apache.hadoop.hbase.HTableDescriptor.constants) +end + +# Include classes definition +require 'hbase/hbase' +require 'hbase/admin' +require 'hbase/table' +require 'hbase/replication_admin' +require 'hbase/security' +require 'hbase/visibility_labels' diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb new file mode 100644 index 0000000000..a8f543726d --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/admin.rb @@ -0,0 +1,973 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import java.util.Arrays +java_import org.apache.hadoop.hbase.util.Pair +java_import org.apache.hadoop.hbase.util.RegionSplitter +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.ServerName +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class Admin + include HBaseConstants + + def initialize(admin, formatter) + @admin = admin + @connection = @admin.getConnection() + @formatter = formatter + end + + def close + @admin.close + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of tables in hbase + def list(regex = ".*") + @admin.listTables(regex).map { |t| t.getNameAsString } + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region flush + def flush(table_or_region_name) + @admin.flush(table_or_region_name) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family compaction + def compact(table_or_region_name, family = nil) + if family == nil + @admin.compact(table_or_region_name) + else + # We are compacting a column family within a region. + @admin.compact(table_or_region_name, family) + end + end + + # Requests to compact all regions on the regionserver + def compact_regionserver(servername, major = false) + @admin.compactRegionServer(ServerName.valueOf(servername), major) + end + + #---------------------------------------------------------------------------------------------- + # Requests a table or region or column family major compaction + def major_compact(table_or_region_name, family = nil) + if family == nil + @admin.majorCompact(table_or_region_name) + else + # We are major compacting a column family within a region or table. + @admin.majorCompact(table_or_region_name, family) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a regionserver's WAL roll + def wal_roll(server_name) + @admin.rollWALWriter(ServerName.valueOf(server_name)) + end + # TODO remove older hlog_roll version + alias :hlog_roll :wal_roll + + #---------------------------------------------------------------------------------------------- + # Requests a table or region split + def split(table_or_region_name, split_point) + if split_point == nil + @admin.split(table_or_region_name) + else + @admin.split(table_or_region_name, split_point) + end + end + + #---------------------------------------------------------------------------------------------- + # Requests a cluster balance + # Returns true if balancer ran + def balancer() + @admin.balancer() + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable balancer + # Returns previous balancer switch setting. + def balance_switch(enableDisable) + @admin.setBalancerRunning( + java.lang.Boolean::valueOf(enableDisable), java.lang.Boolean::valueOf(false)) + end + + #---------------------------------------------------------------------------------------------- + # Request a scan of the catalog table (for garbage collection) + # Returns an int signifying the number of entries cleaned + def catalogjanitor_run() + @admin.runCatalogScan() + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable the catalog janitor + # Returns previous catalog janitor switch setting. + def catalogjanitor_switch(enableDisable) + @admin.enableCatalogJanitor(java.lang.Boolean::valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query on the catalog janitor state (enabled/disabled?) + # Returns catalog janitor state (true signifies enabled). + def catalogjanitor_enabled() + @admin.isCatalogJanitorEnabled() + end + + #---------------------------------------------------------------------------------------------- + # Enables a table + def enable(table_name) + tableExists(table_name) + return if enabled?(table_name) + @admin.enableTable(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Enables all tables matching the given regex + def enable_all(regex) + regex = regex.to_s + @admin.enableTables(regex) + end + + #---------------------------------------------------------------------------------------------- + # Disables a table + def disable(table_name) + tableExists(table_name) + return if disabled?(table_name) + @admin.disableTable(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Disables all tables matching the given regex + def disable_all(regex) + regex = regex.to_s + @admin.disableTables(regex).map { |t| t.getTableName().getNameAsString } + end + + #--------------------------------------------------------------------------------------------- + # Throw exception if table doesn't exist + def tableExists(table_name) + raise ArgumentError, "Table #{table_name} does not exist." unless exists?(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Is table disabled? + def disabled?(table_name) + @admin.isTableDisabled(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop(table_name) + tableExists(table_name) + raise ArgumentError, "Table #{table_name} is enabled. Disable it first." if enabled?(table_name) + + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + end + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop_all(regex) + regex = regex.to_s + failed = @admin.deleteTables(regex).map { |t| t.getTableName().getNameAsString } + return failed + end + + #---------------------------------------------------------------------------------------------- + # Returns ZooKeeper status dump + def zk_dump + @zk_wrapper = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new( + @admin.getConfiguration(), + "admin", + nil) + zk = @zk_wrapper.getRecoverableZooKeeper().getZooKeeper() + @zk_main = org.apache.zookeeper.ZooKeeperMain.new(zk) + org.apache.hadoop.hbase.zookeeper.ZKUtil::dump(@zk_wrapper) + end + + #---------------------------------------------------------------------------------------------- + # Creates a table + def create(table_name, *args) + # Fail if table name is not a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Flatten params array + args = args.flatten.compact + has_columns = false + + # Start defining the table + htd = org.apache.hadoop.hbase.HTableDescriptor.new(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + splits = nil + # Args are either columns or splits, add them to the table definition + # TODO: add table options support + args.each do |arg| + unless arg.kind_of?(String) || arg.kind_of?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + + # First, handle all the cases where arg is a column family. + if arg.kind_of?(String) or arg.has_key?(NAME) + # If the arg is a string, default action is to add a column to the table. + # If arg has a name, it must also be a column descriptor. + descriptor = hcd(arg, htd); + # Warn if duplicate columns are added + if htd.hasFamily(descriptor.getName) + puts "Family '" + descriptor.getNameAsString() + "' already exists, the old one will be replaced" + htd.modifyFamily(descriptor) + else + htd.addFamily(descriptor) + end + has_columns = true + next + end + if arg.has_key?(REGION_REPLICATION) + region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION)) + htd.setRegionReplication(region_replication) + end + + # Get rid of the "METHOD", which is deprecated for create. + # We'll do whatever it used to do below if it's table_att. + if (method = arg.delete(METHOD)) + raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' + end + + # The hash is not a column family. Figure out what's in it. + # First, handle splits. + if arg.has_key?(SPLITS_FILE) + splits_file = arg.delete(SPLITS_FILE) + unless File.exist?(splits_file) + raise(ArgumentError, "Splits file #{splits_file} doesn't exist") + end + arg[SPLITS] = [] + File.foreach(splits_file) do |line| + arg[SPLITS].push(line.strip()) + end + htd.setValue(SPLITS_FILE, arg[SPLITS_FILE]) + end + + if arg.has_key?(SPLITS) + splits = Java::byte[][arg[SPLITS].size].new + idx = 0 + arg.delete(SPLITS).each do |split| + splits[idx] = org.apache.hadoop.hbase.util.Bytes.toBytesBinary(split) + idx = idx + 1 + end + elsif arg.has_key?(NUMREGIONS) or arg.has_key?(SPLITALGO) + # deprecated region pre-split API; if one of the above is specified, will be ignored. + raise(ArgumentError, "Number of regions must be specified") unless arg.has_key?(NUMREGIONS) + raise(ArgumentError, "Split algorithm must be specified") unless arg.has_key?(SPLITALGO) + raise(ArgumentError, "Number of regions must be greater than 1") unless arg[NUMREGIONS] > 1 + num_regions = arg.delete(NUMREGIONS) + split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) + splits = split_algo.split(JInteger.valueOf(num_regions)) + end + + # Done with splits; apply formerly-table_att parameters. + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] + htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] + htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] + htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] + # DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. + # However, it has to be set before DURABILITY so that DURABILITY could overwrite if both args are set + if arg.include?(DEFERRED_LOG_FLUSH) + if arg.delete(DEFERRED_LOG_FLUSH).to_s.upcase == "TRUE" + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("ASYNC_WAL")) + else + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("SYNC_WAL")) + end + end + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] + set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] + set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] + + arg.each_key do |ignored_key| + puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ]) + end + end + + # Fail if no column families defined + raise(ArgumentError, "Table must have at least one column family") if !has_columns + + if splits.nil? + # Perform the create table call + @admin.createTable(htd) + else + # Perform the create table call + @admin.createTable(htd, splits) + end + end + + #---------------------------------------------------------------------------------------------- + # Closes a region. + # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). + # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) + def close_region(region_name, server) + if (server == nil || !closeEncodedRegion?(region_name, server)) + @admin.closeRegion(region_name, server) + end + end + + #---------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------- + # Assign a region + def assign(region_name) + @admin.assign(region_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Unassign a region + def unassign(region_name, force) + @admin.unassign(region_name.to_java_bytes, java.lang.Boolean::valueOf(force)) + end + + #---------------------------------------------------------------------------------------------- + # Move a region + def move(encoded_region_name, server = nil) + @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) + end + + #---------------------------------------------------------------------------------------------- + # Merge two regions + def merge_region(encoded_region_a_name, encoded_region_b_name, force) + @admin.mergeRegions(encoded_region_a_name.to_java_bytes, encoded_region_b_name.to_java_bytes, java.lang.Boolean::valueOf(force)) + end + + #---------------------------------------------------------------------------------------------- + # Returns table's structure description + def describe(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).to_s + end + + def get_column_families(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).getColumnFamilies() + end + + def get_table_attributes(table_name) + @admin.getTableDescriptor(TableName.valueOf(table_name)).toStringTableAttributes + end + + #---------------------------------------------------------------------------------------------- + # Truncates table (deletes all records by recreating the table) + def truncate(table_name, conf = @conf) + table_description = @admin.getTableDescriptor(TableName.valueOf(table_name)) + raise ArgumentError, "Table #{table_name} is not enabled. Enable it first." unless enabled?(table_name) + yield 'Disabling table...' if block_given? + @admin.disableTable(table_name) + + begin + yield 'Truncating table...' if block_given? + @admin.truncateTable(org.apache.hadoop.hbase.TableName.valueOf(table_name), false) + rescue => e + # Handle the compatibility case, where the truncate method doesn't exists on the Master + raise e unless e.respond_to?(:cause) && e.cause != nil + rootCause = e.cause + if rootCause.kind_of?(org.apache.hadoop.hbase.DoNotRetryIOException) then + # Handle the compatibility case, where the truncate method doesn't exists on the Master + yield 'Dropping table...' if block_given? + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + + yield 'Creating table...' if block_given? + @admin.createTable(table_description) + else + raise e + end + end + end + + #---------------------------------------------------------------------------------------------- + # Truncates table while maintaing region boundaries (deletes all records by recreating the table) + def truncate_preserve(table_name, conf = @conf) + h_table = @connection.getTable(TableName.valueOf(table_name)) + locator = @connection.getRegionLocator(TableName.valueOf(table_name)) + splits = locator.getAllRegionLocations(). + map{|i| Bytes.toString(i.getRegionInfo().getStartKey)}. + delete_if{|k| k == ""}.to_java :String + locator.close() + + table_description = @admin.getTableDescriptor(TableName.valueOf(table_name)) + yield 'Disabling table...' if block_given? + disable(table_name) + + begin + yield 'Truncating table...' if block_given? + @admin.truncateTable(org.apache.hadoop.hbase.TableName.valueOf(table_name), true) + rescue => e + # Handle the compatibility case, where the truncate method doesn't exists on the Master + raise e unless e.respond_to?(:cause) && e.cause != nil + rootCause = e.cause + if rootCause.kind_of?(org.apache.hadoop.hbase.DoNotRetryIOException) then + # Handle the compatibility case, where the truncate method doesn't exists on the Master + yield 'Dropping table...' if block_given? + @admin.deleteTable(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + + yield 'Creating table with region boundaries...' if block_given? + @admin.createTable(table_description, splits) + else + raise e + end + end + end + + #---------------------------------------------------------------------------------------------- + # Check the status of alter command (number of regions reopened) + def alter_status(table_name) + # Table name should be a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + status = Pair.new() + begin + status = @admin.getAlterStatus(org.apache.hadoop.hbase.TableName.valueOf(table_name)) + if status.getSecond() != 0 + puts "#{status.getSecond() - status.getFirst()}/#{status.getSecond()} regions updated." + else + puts "All regions updated." + end + sleep 1 + end while status != nil && status.getFirst() != 0 + puts "Done." + end + + #---------------------------------------------------------------------------------------------- + # Change table structure or table options + def alter(table_name, wait = true, *args) + # Table name should be a string + raise(ArgumentError, "Table name must be of type String") unless table_name.kind_of?(String) + + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + # There should be at least one argument + raise(ArgumentError, "There should be at least one argument but the table name") if args.empty? + + # Get table descriptor + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) + + # Process all args + args.each do |arg| + + + # Normalize args to support column name only alter specs + arg = { NAME => arg } if arg.kind_of?(String) + + # Normalize args to support shortcut delete syntax + arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] + + # There are 3 possible options. + # 1) Column family spec. Distinguished by having a NAME and no METHOD. + method = arg.delete(METHOD) + if method == nil and arg.has_key?(NAME) + descriptor = hcd(arg, htd) + column_name = descriptor.getNameAsString + + # If column already exist, then try to alter it. Create otherwise. + if htd.hasFamily(column_name.to_java_bytes) + @admin.modifyColumn(table_name, descriptor) + else + @admin.addColumn(table_name, descriptor) + end + + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + + # We bypass descriptor when adding column families; refresh it to apply other args correctly. + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) + next + end + + # 2) Method other than table_att, with some args. + name = arg.delete(NAME) + if method != nil and method != "table_att" + # Delete column family + if method == "delete" + raise(ArgumentError, "NAME parameter missing for delete method") unless name + @admin.deleteColumn(table_name, name) + # Unset table attributes + elsif method == "table_att_unset" + raise(ArgumentError, "NAME parameter missing for table_att_unset method") unless name + if name.kind_of?(Array) + name.each do |key| + if (htd.getValue(key) == nil) + raise ArgumentError, "Could not find attribute: #{key}" + end + htd.remove(key) + end + else + if (htd.getValue(name) == nil) + raise ArgumentError, "Could not find attribute: #{name}" + end + htd.remove(name) + end + @admin.modifyTable(table_name.to_java_bytes, htd) + # Unknown method + else + raise ArgumentError, "Unknown method: #{method}" + end + + arg.each_key do |unknown_key| + puts("Unknown argument ignored: %s" % [unknown_key]) + end + + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + + if method == "delete" + # We bypass descriptor when deleting column families; refresh it to apply other args correctly. + htd = @admin.getTableDescriptor(TableName.valueOf(table_name)) + end + next + end + + # 3) Some args for the table, optionally with METHOD => table_att (deprecated) + raise(ArgumentError, "NAME argument in an unexpected place") if name + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] + htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] + htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] + htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] + # DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. + # However, it has to be set before DURABILITY so that DURABILITY could overwrite if both args are set + if arg.include?(DEFERRED_LOG_FLUSH) + if arg.delete(DEFERRED_LOG_FLUSH).to_s.upcase == "TRUE" + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("ASYNC_WAL")) + else + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf("SYNC_WAL")) + end + end + htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] + htd.setRegionReplication(JInteger.valueOf(arg.delete(REGION_REPLICATION))) if arg[REGION_REPLICATION] + set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] + set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] + + # set a coprocessor attribute + valid_coproc_keys = [] + if arg.kind_of?(Hash) + arg.each do |key, value| + k = String.new(key) # prepare to strip + k.strip! + + if (k =~ /coprocessor/i) + v = String.new(value) + v.strip! + htd.addCoprocessor(v) + valid_coproc_keys << key + end + end + + valid_coproc_keys.each do |key| + arg.delete(key) + end + + @admin.modifyTable(table_name.to_java_bytes, htd) + + arg.each_key do |unknown_key| + puts("Unknown argument ignored: %s" % [unknown_key]) + end + + if wait == true + puts "Updating all regions with the new schema..." + alter_status(table_name) + end + next + end + end + end + + def status(format, type) + status = @admin.getClusterStatus() + if format == "detailed" + puts("version %s" % [ status.getHBaseVersion() ]) + # Put regions in transition first because usually empty + puts("%d regionsInTransition" % status.getRegionsInTransition().size()) + for k, v in status.getRegionsInTransition() + puts(" %s" % [v]) + end + master_coprocs = java.util.Arrays.toString(@admin.getMasterCoprocessors()) + if master_coprocs != nil + puts("master coprocessors: %s" % master_coprocs) + end + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + puts(" %s:%d %d" % \ + [ server.getHostname(), server.getPort(), server.getStartcode() ]) + puts(" %s" % [ status.getLoad(server).toString() ]) + for name, region in status.getLoad(server).getRegionsLoad() + puts(" %s" % [ region.getNameAsString().dump ]) + puts(" %s" % [ region.toString() ]) + end + end + puts("%d dead servers" % [ status.getDeadServers() ]) + for server in status.getDeadServerNames() + puts(" %s" % [ server ]) + end + elsif format == "replication" + #check whether replication is enabled or not + if (!@admin.getConfiguration().getBoolean(org.apache.hadoop.hbase.HConstants::REPLICATION_ENABLE_KEY, + org.apache.hadoop.hbase.HConstants::REPLICATION_ENABLE_DEFAULT)) + puts("Please enable replication first.") + else + puts("version %s" % [ status.getHBaseVersion() ]) + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + sl = status.getLoad(server) + rSinkString = " SINK :" + rSourceString = " SOURCE:" + rLoadSink = sl.getReplicationLoadSink() + rSinkString << " AgeOfLastAppliedOp=" + rLoadSink.getAgeOfLastAppliedOp().to_s + rSinkString << ", TimeStampsOfLastAppliedOp=" + + (java.util.Date.new(rLoadSink.getTimeStampsOfLastAppliedOp())).toString() + rLoadSourceList = sl.getReplicationLoadSourceList() + index = 0 + while index < rLoadSourceList.size() + rLoadSource = rLoadSourceList.get(index) + rSourceString << " PeerID=" + rLoadSource.getPeerID() + rSourceString << ", AgeOfLastShippedOp=" + rLoadSource.getAgeOfLastShippedOp().to_s + rSourceString << ", SizeOfLogQueue=" + rLoadSource.getSizeOfLogQueue().to_s + rSourceString << ", TimeStampsOfLastShippedOp=" + + (java.util.Date.new(rLoadSource.getTimeStampOfLastShippedOp())).toString() + rSourceString << ", Replication Lag=" + rLoadSource.getReplicationLag().to_s + index = index + 1 + end + puts(" %s:" % + [ server.getHostname() ]) + if type.casecmp("SOURCE") == 0 + puts("%s" % rSourceString) + elsif type.casecmp("SINK") == 0 + puts("%s" % rSinkString) + else + puts("%s" % rSourceString) + puts("%s" % rSinkString) + end + end + end + elsif format == "simple" + load = 0 + regions = 0 + puts("%d live servers" % [ status.getServersSize() ]) + for server in status.getServers() + puts(" %s:%d %d" % \ + [ server.getHostname(), server.getPort(), server.getStartcode() ]) + puts(" %s" % [ status.getLoad(server).toString() ]) + load += status.getLoad(server).getNumberOfRequests() + regions += status.getLoad(server).getNumberOfRegions() + end + puts("%d dead servers" % [ status.getDeadServers() ]) + for server in status.getDeadServerNames() + puts(" %s" % [ server ]) + end + puts("Aggregate load: %d, regions: %d" % [ load , regions ] ) + else + puts "#{status.getServersSize} servers, #{status.getDeadServers} dead, #{'%.4f' % status.getAverageLoad} average load" + end + end + + #---------------------------------------------------------------------------------------------- + # + # Helper methods + # + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + + #---------------------------------------------------------------------------------------------- + # Is table enabled + def enabled?(table_name) + @admin.isTableEnabled(table_name) + end + + #---------------------------------------------------------------------------------------------- + #Is supplied region name is encoded region name + def closeEncodedRegion?(region_name, server) + @admin.closeRegionWithEncodedRegionName(region_name, server) + end + + #---------------------------------------------------------------------------------------------- + # Return a new HColumnDescriptor made of passed args + def hcd(arg, htd) + # String arg, single parameter constructor + return org.apache.hadoop.hbase.HColumnDescriptor.new(arg) if arg.kind_of?(String) + + raise(ArgumentError, "Column family #{arg} must have a name") unless name = arg.delete(NAME) + + family = htd.getFamily(name.to_java_bytes) + # create it if it's a new family + family ||= org.apache.hadoop.hbase.HColumnDescriptor.new(name.to_java_bytes) + + family.setBlockCacheEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKCACHE) + family.setScope(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::REPLICATION_SCOPE) + family.setCacheDataOnWrite(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::CACHE_DATA_ON_WRITE) + family.setInMemory(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY) + family.setTimeToLive(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::TTL))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL) + family.setDataBlockEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING) + family.setBlocksize(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) + family.setMaxVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) + family.setMinVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) + family.setKeepDeletedCells(org.apache.hadoop.hbase.KeepDeletedCells.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS).to_s.upcase)) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + family.setCompressTags(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS) + family.setPrefetchBlocksOnOpen(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN) + family.setValue(COMPRESSION_COMPACT, arg.delete(COMPRESSION_COMPACT)) if arg.include?(COMPRESSION_COMPACT) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) + bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase + unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) + else + family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION) + compression = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION).upcase + unless org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.include?(compression) + raise(ArgumentError, "Compression #{compression} is not supported. Use one of " + org.apache.hadoop.hbase.io.compress.Compression::Algorithm.constants.join(" ")) + else + family.setCompressionType(org.apache.hadoop.hbase.io.compress.Compression::Algorithm.valueOf(compression)) + end + end + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION) + algorithm = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION).upcase + family.setEncryptionType(algorithm) + if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY) + key = org.apache.hadoop.hbase.io.crypto.Encryption.pbkdf128( + arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::ENCRYPTION_KEY)) + family.setEncryptionKey(org.apache.hadoop.hbase.security.EncryptionUtil.wrapKey(@conf, key, + algorithm)) + end + end + + set_user_metadata(family, arg.delete(METADATA)) if arg[METADATA] + set_descriptor_config(family, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] + + arg.each_key do |unknown_key| + puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key]) + end + + return family + end + + #---------------------------------------------------------------------------------------------- + # Enables/disables a region by name + def online(region_name, on_off) + # Open meta table + meta = @connection.getTable(org.apache.hadoop.hbase.TableName::META_TABLE_NAME) + + # Read region info + # FIXME: fail gracefully if can't find the region + region_bytes = region_name.to_java_bytes + g = org.apache.hadoop.hbase.client.Get.new(region_bytes) + g.addColumn(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER) + hri_bytes = meta.get(g).value + + # Change region status + hri = org.apache.hadoop.hbase.util.Writables.getWritable(hri_bytes, org.apache.hadoop.hbase.HRegionInfo.new) + hri.setOffline(on_off) + + # Write it back + put = org.apache.hadoop.hbase.client.Put.new(region_bytes) + put.add(org.apache.hadoop.hbase.HConstants::CATALOG_FAMILY, org.apache.hadoop.hbase.HConstants::REGIONINFO_QUALIFIER, org.apache.hadoop.hbase.util.Writables.getBytes(hri)) + meta.put(put) + end + # Apply user metadata to table/column descriptor + def set_user_metadata(descriptor, metadata) + raise(ArgumentError, "#{METADATA} must be a Hash type") unless metadata.kind_of?(Hash) + for k,v in metadata + v = v.to_s unless v.nil? + descriptor.setValue(k, v) + end + end + + #---------------------------------------------------------------------------------------------- + # Take a snapshot of specified table + def snapshot(table, snapshot_name, *args) + if args.empty? + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + else + args.each do |arg| + if arg[SKIP_FLUSH] == true + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes, SnapshotDescription::Type::SKIPFLUSH) + else + @admin.snapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + end + end + end + + #---------------------------------------------------------------------------------------------- + # Restore specified snapshot + def restore_snapshot(snapshot_name) + @admin.restoreSnapshot(snapshot_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Create a new table by cloning the snapshot content + def clone_snapshot(snapshot_name, table) + @admin.cloneSnapshot(snapshot_name.to_java_bytes, table.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Delete specified snapshot + def delete_snapshot(snapshot_name) + @admin.deleteSnapshot(snapshot_name.to_java_bytes) + end + + #---------------------------------------------------------------------------------------------- + # Deletes the snapshots matching the given regex + def delete_all_snapshot(regex) + @admin.deleteSnapshots(regex).to_a + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of snapshots + def list_snapshot(regex = ".*") + @admin.listSnapshots(regex).to_a + end + + # Apply config specific to a table/column to its descriptor + def set_descriptor_config(descriptor, config) + raise(ArgumentError, "#{CONFIGURATION} must be a Hash type") unless config.kind_of?(Hash) + for k,v in config + v = v.to_s unless v.nil? + descriptor.setConfiguration(k, v) + end + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of one regionserver. + def update_config(serverName) + @admin.updateConfiguration(ServerName.valueOf(serverName)); + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers. + def update_all_config() + @admin.updateConfiguration(); + end + + #---------------------------------------------------------------------------------------------- + # Returns namespace's structure description + def describe_namespace(namespace_name) + namespace = @admin.getNamespaceDescriptor(namespace_name) + + unless namespace.nil? + return namespace.to_s + end + + raise(ArgumentError, "Failed to find namespace named #{namespace_name}") + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of namespaces in hbase + def list_namespace(regex = ".*") + pattern = java.util.regex.Pattern.compile(regex) + list = @admin.listNamespaceDescriptors.map { |ns| ns.getName } + list.select {|s| pattern.match(s) } + end + + #---------------------------------------------------------------------------------------------- + # Returns a list of tables in namespace + def list_namespace_tables(namespace_name) + unless namespace_name.nil? + return @admin.listTableNamesByNamespace(namespace_name).map { |t| t.getQualifierAsString() } + end + + raise(ArgumentError, "Failed to find namespace named #{namespace_name}") + end + + #---------------------------------------------------------------------------------------------- + # Creates a namespace + def create_namespace(namespace_name, *args) + # Fail if table name is not a string + raise(ArgumentError, "Namespace name must be of type String") unless namespace_name.kind_of?(String) + + # Flatten params array + args = args.flatten.compact + + # Start defining the table + nsb = org.apache.hadoop.hbase.NamespaceDescriptor::create(namespace_name) + args.each do |arg| + unless arg.kind_of?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") + end + for k,v in arg + v = v.to_s unless v.nil? + nsb.addConfiguration(k, v) + end + end + @admin.createNamespace(nsb.build()); + end + + #---------------------------------------------------------------------------------------------- + # modify a namespace + def alter_namespace(namespace_name, *args) + # Fail if table name is not a string + raise(ArgumentError, "Namespace name must be of type String") unless namespace_name.kind_of?(String) + + nsd = @admin.getNamespaceDescriptor(namespace_name) + + unless nsd + raise(ArgumentError, "Namespace does not exist") + end + nsb = org.apache.hadoop.hbase.NamespaceDescriptor::create(nsd) + + # Flatten params array + args = args.flatten.compact + + # Start defining the table + args.each do |arg| + unless arg.kind_of?(Hash) + raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash type") + end + method = arg[METHOD] + if method == "unset" + nsb.removeConfiguration(arg[NAME]) + elsif method == "set" + arg.delete(METHOD) + for k,v in arg + v = v.to_s unless v.nil? + + nsb.addConfiguration(k, v) + end + else + raise(ArgumentError, "Unknown method #{method}") + end + end + @admin.modifyNamespace(nsb.build()); + end + + + #---------------------------------------------------------------------------------------------- + # Drops a table + def drop_namespace(namespace_name) + @admin.deleteNamespace(namespace_name) + end + + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb new file mode 100644 index 0000000000..89700a486b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/hbase.rb @@ -0,0 +1,70 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +require 'hbase/admin' +require 'hbase/table' +require 'hbase/security' +require 'hbase/visibility_labels' + +module Hbase + class Hbase + attr_accessor :configuration + + def initialize(config = nil) + # Create configuration + if config + self.configuration = config + else + self.configuration = org.apache.hadoop.hbase.HBaseConfiguration.create + # Turn off retries in hbase and ipc. Human doesn't want to wait on N retries. + configuration.setInt("hbase.client.retries.number", 7) + configuration.setInt("hbase.ipc.client.connect.max.retries", 3) + end + @connection = org.apache.hadoop.hbase.client.ConnectionFactory.createConnection( + self.configuration) + end + + def admin(formatter) + ::Hbase::Admin.new(@connection.getAdmin, formatter) + end + + # Create new one each time + def table(table, shell) + ::Hbase::Table.new(@connection.getTable(table), shell) + end + + def replication_admin(formatter) + ::Hbase::RepAdmin.new(configuration, formatter) + end + + def security_admin(formatter) + ::Hbase::SecurityAdmin.new(@connection.getAdmin, formatter) + end + + def visibility_labels_admin(formatter) + ::Hbase::VisibilityLabelsAdmin.new(@connection.getAdmin, formatter) + end + + def shutdown + @connection.close + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb new file mode 100644 index 0000000000..617073b6a0 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/replication_admin.rb @@ -0,0 +1,174 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +java_import org.apache.hadoop.hbase.client.replication.ReplicationAdmin +java_import org.apache.hadoop.hbase.replication.ReplicationPeerConfig +java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.zookeeper.ZKUtil +java_import org.apache.hadoop.hbase.TableName + +# Wrapper for org.apache.hadoop.hbase.client.replication.ReplicationAdmin + +module Hbase + class RepAdmin + include HBaseConstants + + def initialize(configuration, formatter) + @replication_admin = ReplicationAdmin.new(configuration) + @configuration = configuration + @formatter = formatter + end + + #---------------------------------------------------------------------------------------------- + # Add a new peer cluster to replicate to + def add_peer(id, args = {}, peer_tableCFs = nil) + # make add_peer backwards compatible to take in string for clusterKey and peer_tableCFs + if args.is_a?(String) + cluster_key = args + @replication_admin.addPeer(id, cluster_key, peer_tableCFs) + elsif args.is_a?(Hash) + unless peer_tableCFs.nil? + raise(ArgumentError, "peer_tableCFs should be specified as TABLE_CFS in args") + end + + endpoint_classname = args.fetch(ENDPOINT_CLASSNAME, nil) + cluster_key = args.fetch(CLUSTER_KEY, nil) + + # Handle cases where custom replication endpoint and cluster key are either both provided + # or neither are provided + if endpoint_classname.nil? and cluster_key.nil? + raise(ArgumentError, "Either ENDPOINT_CLASSNAME or CLUSTER_KEY must be specified.") + elsif !endpoint_classname.nil? and !cluster_key.nil? + raise(ArgumentError, "ENDPOINT_CLASSNAME and CLUSTER_KEY cannot both be specified.") + end + + # Cluster Key is required for ReplicationPeerConfig for a custom replication endpoint + if !endpoint_classname.nil? and cluster_key.nil? + cluster_key = ZKUtil.getZooKeeperClusterKey(@configuration) + end + + # Optional parameters + config = args.fetch(CONFIG, nil) + data = args.fetch(DATA, nil) + table_cfs = args.fetch(TABLE_CFS, nil) + + # Create and populate a ReplicationPeerConfig + replication_peer_config = ReplicationPeerConfig.new + replication_peer_config.set_cluster_key(cluster_key) + + unless endpoint_classname.nil? + replication_peer_config.set_replication_endpoint_impl(endpoint_classname) + end + + unless config.nil? + replication_peer_config.get_configuration.put_all(config) + end + + unless data.nil? + # Convert Strings to Bytes for peer_data + peer_data = replication_peer_config.get_peer_data + data.each{|key, val| + peer_data.put(Bytes.to_bytes(key), Bytes.to_bytes(val)) + } + end + + @replication_admin.add_peer(id, replication_peer_config, table_cfs) + else + raise(ArgumentError, "args must be either a String or Hash") + end + end + + #---------------------------------------------------------------------------------------------- + # Remove a peer cluster, stops the replication + def remove_peer(id) + @replication_admin.removePeer(id) + end + + + #--------------------------------------------------------------------------------------------- + # Show replcated tables/column families, and their ReplicationType + def list_replicated_tables(regex = ".*") + pattern = java.util.regex.Pattern.compile(regex) + list = @replication_admin.listReplicated() + list.select {|s| pattern.match(s.get(ReplicationAdmin::TNAME))} + end + + #---------------------------------------------------------------------------------------------- + # List all peer clusters + def list_peers + @replication_admin.listPeers + end + + #---------------------------------------------------------------------------------------------- + # Get peer cluster state + def get_peer_state(id) + @replication_admin.getPeerState(id) ? "ENABLED" : "DISABLED" + end + + #---------------------------------------------------------------------------------------------- + # Restart the replication stream to the specified peer + def enable_peer(id) + @replication_admin.enablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Stop the replication stream to the specified peer + def disable_peer(id) + @replication_admin.disablePeer(id) + end + + #---------------------------------------------------------------------------------------------- + # Show the current tableCFs config for the specified peer + def show_peer_tableCFs(id) + @replication_admin.getPeerTableCFs(id) + end + + #---------------------------------------------------------------------------------------------- + # Set new tableCFs config for the specified peer + def set_peer_tableCFs(id, tableCFs) + @replication_admin.setPeerTableCFs(id, tableCFs) + end + + #---------------------------------------------------------------------------------------------- + # Append a tableCFs config for the specified peer + def append_peer_tableCFs(id, tableCFs) + @replication_admin.appendPeerTableCFs(id, tableCFs) + end + + #---------------------------------------------------------------------------------------------- + # Remove some tableCFs from the tableCFs config of the specified peer + def remove_peer_tableCFs(id, tableCFs) + @replication_admin.removePeerTableCFs(id, tableCFs) + end + #---------------------------------------------------------------------------------------------- + # Enables a table's replication switch + def enable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @replication_admin.enableTableRep(tableName) + end + #---------------------------------------------------------------------------------------------- + # Disables a table's replication switch + def disable_tablerep(table_name) + tableName = TableName.valueOf(table_name) + @replication_admin.disableTableRep(tableName) + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb new file mode 100644 index 0000000000..bfdd7330ff --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/security.rb @@ -0,0 +1,190 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin + +module Hbase + class SecurityAdmin + include HBaseConstants + + def initialize(admin, formatter) + @admin = admin + @config = @admin.getConfiguration() + @formatter = formatter + end + + def close + @admin.close + end + + #---------------------------------------------------------------------------------------------- + def grant(user, permissions, table_name=nil, family=nil, qualifier=nil) + security_available? + + # TODO: need to validate user name + + begin + # Verify that the specified permission is valid + if (permissions == nil || permissions.length == 0) + raise(ArgumentError, "Invalid permission: no actions associated with user") + end + + perm = org.apache.hadoop.hbase.security.access.Permission.new( + permissions.to_java_bytes) + + if (table_name != nil) + tablebytes=table_name.to_java_bytes + #check if the tablename passed is actually a namespace + if (isNamespace?(table_name)) + # Namespace should exist first. + namespace_name = table_name[1...table_name.length] + raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless + namespace_exists?(namespace_name) + + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, namespace_name, user, perm.getActions()) + else + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name.to_java_bytes) + htd = @admin.getTableDescriptor(tableName) + + if (family != nil) + raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + fambytes = family.to_java_bytes if (family != nil) + qualbytes = qualifier.to_java_bytes if (qualifier != nil) + + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, tableName, user, fambytes, qualbytes, perm.getActions()) + end + else + # invoke cp endpoint to perform access controls + org.apache.hadoop.hbase.security.access.AccessControlClient.grant( + @config, user, perm.getActions()) + end + end + end + + #---------------------------------------------------------------------------------------------- + def revoke(user, table_name=nil, family=nil, qualifier=nil) + security_available? + + # TODO: need to validate user name + + begin + if (table_name != nil) + #check if the tablename passed is actually a namespace + if (isNamespace?(table_name)) + # Namespace should exist first. + namespace_name = table_name[1...table_name.length] + raise(ArgumentError, "Can't find a namespace: #{namespace_name}") unless namespace_exists?(namespace_name) + + tablebytes=table_name.to_java_bytes + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, namespace_name, user) + else + # Table should exist + raise(ArgumentError, "Can't find a table: #{table_name}") unless exists?(table_name) + + tableName = org.apache.hadoop.hbase.TableName.valueOf(table_name.to_java_bytes) + htd = @admin.getTableDescriptor(tableName) + + if (family != nil) + raise(ArgumentError, "Can't find a family: #{family}") unless htd.hasFamily(family.to_java_bytes) + end + + fambytes = family.to_java_bytes if (family != nil) + qualbytes = qualifier.to_java_bytes if (qualifier != nil) + + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, tableName, user, fambytes, qualbytes) + end + else + perm = org.apache.hadoop.hbase.security.access.Permission.new(''.to_java_bytes) + org.apache.hadoop.hbase.security.access.AccessControlClient.revoke( + @config, user, perm.getActions()) + end + end + end + + #---------------------------------------------------------------------------------------------- + def user_permission(table_regex=nil) + security_available? + all_perms = org.apache.hadoop.hbase.security.access.AccessControlClient.getUserPermissions(@config,table_regex) + res = {} + count = 0 + all_perms.each do |value| + user_name = String.from_java_bytes(value.getUser) + if (table_regex != nil && isNamespace?(table_regex)) + namespace = table_regex[1...table_regex.length] + else + namespace = (value.getTableName != nil) ? value.getTableName.getNamespaceAsString() : '' + end + table = (value.getTableName != nil) ? value.getTableName.getNameAsString() : '' + family = (value.getFamily != nil) ? + org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : + '' + qualifier = (value.getQualifier != nil) ? + org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getQualifier) : + '' + + action = org.apache.hadoop.hbase.security.access.Permission.new value.getActions + + if block_given? + yield(user_name, "#{namespace},#{table},#{family},#{qualifier}: #{action.to_s}") + else + res[user_name] ||= {} + res[user_name][family + ":" +qualifier] = action + end + count += 1 + end + + return ((block_given?) ? count : res) + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + + def isNamespace?(table_name) + table_name.start_with?('@') + end + + # Does Namespace exist + def namespace_exists?(namespace_name) + namespaceDesc = @admin.getNamespaceDescriptor(namespace_name) + if(namespaceDesc == nil) + return false + else + return true + end + end + + # Make sure that security tables are available + def security_available?() + raise(ArgumentError, "DISABLED: Security features are not available") \ + unless exists?(org.apache.hadoop.hbase.security.access.AccessControlLists::ACL_TABLE_NAME) + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb new file mode 100644 index 0000000000..e895302be8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/table.rb @@ -0,0 +1,687 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java + +# Wrapper for org.apache.hadoop.hbase.client.Table + +module Hbase + class Table + include HBaseConstants + + @@thread_pool = nil + + # Add the command 'name' to table s.t. the shell command also called via 'name' + # and has an internal method also called 'name'. + # + # e.g. name = scan, adds table.scan which calls Scan.scan + def self.add_shell_command(name) + self.add_command(name, name, name) + end + + # add a named command to the table instance + # + # name - name of the command that should added to the table + # (eg. sending 'scan' here would allow you to do table.scan) + # shell_command - name of the command in the shell + # internal_method_name - name of the method in the shell command to forward the call + def self.add_command(name, shell_command, internal_method_name) + method = name.to_sym + self.class_eval do + define_method method do |*args| + @shell.internal_command(shell_command, internal_method_name, self, *args) + end + end + end + + # General help for the table + # class level so we can call it from anywhere + def self.help + return <<-EOF +Help for table-reference commands. + +You can either create a table via 'create' and then manipulate the table via commands like 'put', 'get', etc. +See the standard help information for how to use each of these commands. + +However, as of 0.96, you can also get a reference to a table, on which you can invoke commands. +For instance, you can get create a table and keep around a reference to it via: + + hbase> t = create 't', 'cf' + +Or, if you have already created the table, you can get a reference to it: + + hbase> t = get_table 't' + +You can do things like call 'put' on the table: + + hbase> t.put 'r', 'cf:q', 'v' + +which puts a row 'r' with column family 'cf', qualifier 'q' and value 'v' into table t. + +To read the data out, you can scan the table: + + hbase> t.scan + +which will read all the rows in table 't'. + +Essentially, any command that takes a table name can also be done via table reference. +Other commands include things like: get, delete, deleteall, +get_all_columns, get_counter, count, incr. These functions, along with +the standard JRuby object methods are also available via tab completion. + +For more information on how to use each of these commands, you can also just type: + + hbase> t.help 'scan' + +which will output more information on how to use that command. + +You can also do general admin actions directly on a table; things like enable, disable, +flush and drop just by typing: + + hbase> t.enable + hbase> t.flush + hbase> t.disable + hbase> t.drop + +Note that after dropping a table, your reference to it becomes useless and further usage +is undefined (and not recommended). +EOF + end + + #--------------------------------------------------------------------------------------------- + + # let external objects read the underlying table object + attr_reader :table + # let external objects read the table name + attr_reader :name + + def initialize(table, shell) + @table = table + @name = @table.getName().getNameAsString() + @shell = shell + @converters = Hash.new() + end + + def close() + @table.close() + end + + # Note the below methods are prefixed with '_' to hide them from the average user, as + # they will be much less likely to tab complete to the 'dangerous' internal method + #---------------------------------------------------------------------------------------------- + + # Put a cell 'value' at specified table/row/column + def _put_internal(row, column, value, timestamp = nil, args = {}) + p = org.apache.hadoop.hbase.client.Put.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if args.any? + attributes = args[ATTRIBUTES] + set_attributes(p, attributes) if attributes + visibility = args[VISIBILITY] + set_cell_visibility(p, visibility) if visibility + ttl = args[TTL] + set_op_ttl(p, ttl) if ttl + end + #Case where attributes are specified without timestamp + if timestamp.kind_of?(Hash) + timestamp.each do |k, v| + if k == 'ATTRIBUTES' + set_attributes(p, v) + elsif k == 'VISIBILITY' + set_cell_visibility(p, v) + elsif k == "TTL" + set_op_ttl(p, v) + end + end + timestamp = nil + end + if timestamp + p.add(family, qualifier, timestamp, value.to_s.to_java_bytes) + else + p.add(family, qualifier, value.to_s.to_java_bytes) + end + @table.put(p) + end + + #---------------------------------------------------------------------------------------------- + # Delete a cell + def _delete_internal(row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + _deleteall_internal(row, column, timestamp, args) + end + + #---------------------------------------------------------------------------------------------- + # Delete a row + def _deleteall_internal(row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + # delete operation doesn't need read permission. Retaining the read check for + # meta table as a part of HBASE-5837. + if is_meta_table? + raise ArgumentError, "Row Not Found" if _get_internal(row).nil? + end + temptimestamp = timestamp + if temptimestamp.kind_of?(Hash) + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP + end + d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp) + if temptimestamp.kind_of?(Hash) + temptimestamp.each do |k, v| + if v.kind_of?(String) + set_cell_visibility(d, v) if v + end + end + end + if args.any? + visibility = args[VISIBILITY] + set_cell_visibility(d, visibility) if visibility + end + if column + family, qualifier = parse_column_name(column) + d.deleteColumns(family, qualifier, timestamp) + end + @table.delete(d) + end + + #---------------------------------------------------------------------------------------------- + # Increment a counter atomically + def _incr_internal(row, column, value = nil, args={}) + if value.kind_of?(Hash) + value = 1 + end + value ||= 1 + incr = org.apache.hadoop.hbase.client.Increment.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if qualifier.nil? + raise ArgumentError, "Failed to provide both column family and column qualifier for incr" + end + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(incr, attributes) if attributes + set_cell_visibility(incr, visibility) if visibility + ttl = args[TTL] + set_op_ttl(incr, ttl) if ttl + end + incr.addColumn(family, qualifier, value) + result = @table.increment(incr) + return nil if result.isEmpty + + # Fetch cell value + cell = result.listCells[0] + org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) + end + + #---------------------------------------------------------------------------------------------- + # appends the value atomically + def _append_internal(row, column, value, args={}) + append = org.apache.hadoop.hbase.client.Append.new(row.to_s.to_java_bytes) + family, qualifier = parse_column_name(column) + if qualifier.nil? + raise ArgumentError, "Failed to provide both column family and column qualifier for append" + end + if args.any? + attributes = args[ATTRIBUTES] + visibility = args[VISIBILITY] + set_attributes(append, attributes) if attributes + set_cell_visibility(append, visibility) if visibility + ttl = args[TTL] + set_op_ttl(append, ttl) if ttl + end + append.add(family, qualifier, value.to_s.to_java_bytes) + @table.append(append) + end + + #---------------------------------------------------------------------------------------------- + # Count rows in a table + def _count_internal(interval = 1000, caching_rows = 10) + # We can safely set scanner caching with the first key only filter + scan = org.apache.hadoop.hbase.client.Scan.new + scan.setCacheBlocks(false) + scan.setCaching(caching_rows) + scan.setFilter(org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter.new) + + # Run the scanner + scanner = @table.getScanner(scan) + count = 0 + iter = scanner.iterator + + # Iterate results + while iter.hasNext + row = iter.next + count += 1 + next unless (block_given? && count % interval == 0) + # Allow command modules to visualize counting process + yield(count, + org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) + end + + scanner.close() + # Return the counter + return count + end + + #---------------------------------------------------------------------------------------------- + # Get from table + def _get_internal(row, *args) + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + maxlength = -1 + @converters.clear() + + # Normalize args + args = args.first if args.first.kind_of?(Hash) + if args.kind_of?(String) || args.kind_of?(Array) + columns = [ args ].flatten.compact + args = { COLUMNS => columns } + end + + # + # Parse arguments + # + unless args.kind_of?(Hash) + raise ArgumentError, "Failed parse of of #{args.inspect}, #{args.class}" + end + + # Get maxlength parameter if passed + maxlength = args.delete(MAXLENGTH) if args[MAXLENGTH] + filter = args.delete(FILTER) if args[FILTER] + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args.delete(CONSISTENCY) if args[CONSISTENCY] + replicaId = args.delete(REGION_REPLICA_ID) if args[REGION_REPLICA_ID] + unless args.empty? + columns = args[COLUMN] || args[COLUMNS] + if args[VERSIONS] + vers = args[VERSIONS] + else + vers = 1 + end + if columns + # Normalize types, convert string to an array of strings + columns = [ columns ] if columns.is_a?(String) + + # At this point it is either an array or some unsupported stuff + unless columns.kind_of?(Array) + raise ArgumentError, "Failed parse column argument type #{args.inspect}, #{args.class}" + end + + # Get each column name and add it to the filter + columns.each do |column| + family, qualifier = parse_column_name(column.to_s) + if qualifier + get.addColumn(family, qualifier) + else + get.addFamily(family) + end + end + + # Additional params + get.setMaxVersions(vers) + get.setTimeStamp(args[TIMESTAMP]) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + else + if attributes + set_attributes(get, attributes) + elsif authorizations + set_authorizations(get, authorizations) + else + # May have passed TIMESTAMP and row only; wants all columns from ts. + unless ts = args[TIMESTAMP] || tr = args[TIMERANGE] + raise ArgumentError, "Failed parse of #{args.inspect}, #{args.class}" + end + end + + get.setMaxVersions(vers) + # Set the timestamp/timerange + get.setTimeStamp(ts.to_i) if args[TIMESTAMP] + get.setTimeRange(args[TIMERANGE][0], args[TIMERANGE][1]) if args[TIMERANGE] + end + set_attributes(get, attributes) if attributes + set_authorizations(get, authorizations) if authorizations + end + + unless filter.class == String + get.setFilter(filter) + else + get.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) + end + + get.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency + get.setReplicaId(replicaId) if replicaId + + # Call hbase for the results + result = @table.get(get) + return nil if result.isEmpty + + # Print out results. Result can be Cell or RowResult. + res = {} + result.list.each do |kv| + family = String.from_java_bytes(kv.getFamily) + qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) + + column = "#{family}:#{qualifier}" + value = to_string(column, kv, maxlength) + + if block_given? + yield(column, value) + else + res[column] = value + end + end + + # If block given, we've yielded all the results, otherwise just return them + return ((block_given?) ? nil : res) + end + + #---------------------------------------------------------------------------------------------- + # Fetches and decodes a counter value from hbase + def _get_counter_internal(row, column) + family, qualifier = parse_column_name(column.to_s) + # Format get request + get = org.apache.hadoop.hbase.client.Get.new(row.to_s.to_java_bytes) + get.addColumn(family, qualifier) + get.setMaxVersions(1) + + # Call hbase + result = @table.get(get) + return nil if result.isEmpty + + # Fetch cell value + cell = result.list[0] + org.apache.hadoop.hbase.util.Bytes::toLong(cell.getValue) + end + + def _hash_to_scan(args) + if args.any? + filter = args["FILTER"] + startrow = args["STARTROW"] || '' + stoprow = args["STOPROW"] + timestamp = args["TIMESTAMP"] + columns = args["COLUMNS"] || args["COLUMN"] || [] + # If CACHE_BLOCKS not set, then default 'true'. + cache_blocks = args["CACHE_BLOCKS"].nil? ? true: args["CACHE_BLOCKS"] + cache = args["CACHE"] || 0 + reversed = args["REVERSED"] || false + versions = args["VERSIONS"] || 1 + timerange = args[TIMERANGE] + raw = args["RAW"] || false + attributes = args[ATTRIBUTES] + authorizations = args[AUTHORIZATIONS] + consistency = args[CONSISTENCY] + # Normalize column names + columns = [columns] if columns.class == String + limit = args["LIMIT"] || -1 + unless columns.kind_of?(Array) + raise ArgumentError.new("COLUMNS must be specified as a String or an Array") + end + + scan = if stoprow + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) + else + org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) + end + + columns.each do |c| + family, qualifier = parse_column_name(c.to_s) + if qualifier + scan.addColumn(family, qualifier) + else + scan.addFamily(family) + end + end + + unless filter.class == String + scan.setFilter(filter) + else + scan.setFilter(org.apache.hadoop.hbase.filter.ParseFilter.new.parseFilterString(filter)) + end + + scan.setTimeStamp(timestamp) if timestamp + scan.setCacheBlocks(cache_blocks) + scan.setReversed(reversed) + scan.setCaching(cache) if cache > 0 + scan.setMaxVersions(versions) if versions > 1 + scan.setTimeRange(timerange[0], timerange[1]) if timerange + scan.setRaw(raw) + scan.setCaching(limit) if limit > 0 + set_attributes(scan, attributes) if attributes + set_authorizations(scan, authorizations) if authorizations + scan.setConsistency(org.apache.hadoop.hbase.client.Consistency.valueOf(consistency)) if consistency + else + scan = org.apache.hadoop.hbase.client.Scan.new + end + + scan + end + + def _get_scanner(args) + @table.getScanner(_hash_to_scan(args)) + end + + #---------------------------------------------------------------------------------------------- + # Scans whole table or a range of keys and returns rows matching specific criteria + def _scan_internal(args = {}) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + + limit = args["LIMIT"] || -1 + maxlength = args.delete("MAXLENGTH") || -1 + count = 0 + res = {} + + @converters.clear() + + # Start the scanner + scanner = @table.getScanner(_hash_to_scan(args)) + iter = scanner.iterator + + # Iterate results + while iter.hasNext + row = iter.next + key = org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow) + + row.list.each do |kv| + family = String.from_java_bytes(kv.getFamily) + qualifier = org.apache.hadoop.hbase.util.Bytes::toStringBinary(kv.getQualifier) + + column = "#{family}:#{qualifier}" + cell = to_string(column, kv, maxlength) + + if block_given? + yield(key, "column=#{column}, #{cell}") + else + res[key] ||= {} + res[key][column] = cell + end + end + + # One more row processed + count += 1 + if limit > 0 && count >= limit + # If we reached the limit, exit before the next call to hasNext + break + end + end + + scanner.close() + return ((block_given?) ? count : res) + end + + # Apply OperationAttributes to puts/scans/gets + def set_attributes(oprattr, attributes) + raise(ArgumentError, "Attributes must be a Hash type") unless attributes.kind_of?(Hash) + for k,v in attributes + v = v.to_s unless v.nil? + oprattr.setAttribute(k.to_s, v.to_java_bytes) + end + end + + def set_cell_permissions(op, permissions) + raise(ArgumentError, "Permissions must be a Hash type") unless permissions.kind_of?(Hash) + map = java.util.HashMap.new + permissions.each do |user,perms| + map.put(user.to_s, org.apache.hadoop.hbase.security.access.Permission.new( + perms.to_java_bytes)) + end + op.setACL(map) + end + + def set_cell_visibility(oprattr, visibility) + oprattr.setCellVisibility( + org.apache.hadoop.hbase.security.visibility.CellVisibility.new( + visibility.to_s)) + end + + def set_authorizations(oprattr, authorizations) + raise(ArgumentError, "Authorizations must be a Array type") unless authorizations.kind_of?(Array) + auths = [ authorizations ].flatten.compact + oprattr.setAuthorizations( + org.apache.hadoop.hbase.security.visibility.Authorizations.new( + auths.to_java(:string))) + end + + def set_op_ttl(op, ttl) + op.setTTL(ttl.to_java(:long)) + end + + #---------------------------- + # Add general administration utilities to the shell + # each of the names below adds this method name to the table + # by callling the corresponding method in the shell + # Add single method utilities to the current class + # Generally used for admin functions which just have one name and take the table name + def self.add_admin_utils(*args) + args.each do |method| + define_method method do |*method_args| + @shell.command(method, @name, *method_args) + end + end + end + + #Add the following admin utilities to the table + add_admin_utils :enable, :disable, :flush, :drop, :describe, :snapshot + + #---------------------------- + #give the general help for the table + # or the named command + def help (command = nil) + #if there is a command, get the per-command help from the shell + if command + begin + return @shell.help_command(command) + rescue NoMethodError + puts "Command \'#{command}\' does not exist. Please see general table help." + return nil + end + end + return @shell.help('table_help') + end + + # Table to string + def to_s + cl = self.class() + return "#{cl} - #{@name}" + end + + # Standard ruby call to get the return value for an object + # overriden here so we get sane semantics for printing a table on return + def inspect + to_s + end + + #---------------------------------------------------------------------------------------- + # Helper methods + + # Returns a list of column names in the table + def get_all_columns + @table.table_descriptor.getFamilies.map do |family| + "#{family.getNameAsString}:" + end + end + + # Checks if current table is one of the 'meta' tables + def is_meta_table? + org.apache.hadoop.hbase.TableName::META_TABLE_NAME.equals(@table.getName()) + end + + # Returns family and (when has it) qualifier for a column name + def parse_column_name(column) + split = org.apache.hadoop.hbase.KeyValue.parseColumn(column.to_java_bytes) + set_converter(split) if split.length > 1 + return split[0], (split.length > 1) ? split[1] : nil + end + + # Make a String of the passed kv + # Intercept cells whose format we know such as the info:regioninfo in hbase:meta + def to_string(column, kv, maxlength = -1) + if is_meta_table? + if column == 'info:regioninfo' or column == 'info:splitA' or column == 'info:splitB' + hri = org.apache.hadoop.hbase.HRegionInfo.parseFromOrNull(kv.getValue) + return "timestamp=%d, value=%s" % [kv.getTimestamp, hri.toString] + end + if column == 'info:serverstartcode' + if kv.getValue.length > 0 + str_val = org.apache.hadoop.hbase.util.Bytes.toLong(kv.getValue) + else + str_val = org.apache.hadoop.hbase.util.Bytes.toStringBinary(kv.getValue) + end + return "timestamp=%d, value=%s" % [kv.getTimestamp, str_val] + end + end + + if kv.isDelete + val = "timestamp=#{kv.getTimestamp}, type=#{org.apache.hadoop.hbase.KeyValue::Type::codeToType(kv.getType)}" + else + val = "timestamp=#{kv.getTimestamp}, value=#{convert(column, kv)}" + end + (maxlength != -1) ? val[0, maxlength] : val + end + + def convert(column, kv) + #use org.apache.hadoop.hbase.util.Bytes as the default class + klazz_name = 'org.apache.hadoop.hbase.util.Bytes' + #use org.apache.hadoop.hbase.util.Bytes::toStringBinary as the default convertor + converter = 'toStringBinary' + if @converters.has_key?(column) + # lookup the CONVERTER for certain column - "cf:qualifier" + matches = /c\((.+)\)\.(.+)/.match(@converters[column]) + if matches.nil? + # cannot match the pattern of 'c(className).functionname' + # use the default klazz_name + converter = @converters[column] + else + klazz_name = matches[1] + converter = matches[2] + end + end + method = eval(klazz_name).method(converter) + return method.call(kv.getValue) # apply the converter + end + + # if the column spec contains CONVERTER information, to get rid of :CONVERTER info from column pair. + # 1. return back normal column pair as usual, i.e., "cf:qualifier[:CONVERTER]" to "cf" and "qualifier" only + # 2. register the CONVERTER information based on column spec - "cf:qualifier" + def set_converter(column) + family = String.from_java_bytes(column[0]) + parts = org.apache.hadoop.hbase.KeyValue.parseColumn(column[1]) + if parts.length > 1 + @converters["#{family}:#{String.from_java_bytes(parts[0])}"] = String.from_java_bytes(parts[1]) + column[1] = parts[0] + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb new file mode 100644 index 0000000000..4b99bf9026 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/hbase/visibility_labels.rb @@ -0,0 +1,148 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import org.apache.hadoop.hbase.security.visibility.VisibilityClient +java_import org.apache.hadoop.hbase.security.visibility.VisibilityConstants +java_import org.apache.hadoop.hbase.util.Bytes + +module Hbase + class VisibilityLabelsAdmin + + def initialize(admin, formatter) + @admin = admin + @config = @admin.getConfiguration() + @formatter = formatter + end + + def close + @admin.close + end + + def add_labels(*args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + labels = [ args ].flatten.compact + end + if labels.size() == 0 + raise(ArgumentError, "Arguments cannot be null") + end + + begin + response = VisibilityClient.addLabels(@config, labels.to_java(:string)) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + def set_auths(user, *args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + auths = [ args ].flatten.compact + end + + begin + response = VisibilityClient.setAuths(@config, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + def get_auths(user) + lables_table_available? + begin + response = VisibilityClient.getAuths(@config, user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + return response.getAuthList + end + end + + def list_labels(regex = ".*") + lables_table_available? + begin + response = VisibilityClient.listLabels(@config, regex) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + return response.getLabelList + end + end + + def clear_auths(user, *args) + lables_table_available? + # Normalize args + if args.kind_of?(Array) + auths = [ args ].flatten.compact + end + + begin + response = VisibilityClient.clearAuths(@config, auths.to_java(:string), user) + if response.nil? + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") + end + labelsWithException = "" + list = response.getResultList() + list.each do |result| + if result.hasException() + labelsWithException += Bytes.toString(result.getException().getValue().toByteArray()) + end + end + if labelsWithException.length > 0 + raise(ArgumentError, labelsWithException) + end + end + end + + # Make sure that lables table is available + def lables_table_available?() + raise(ArgumentError, "DISABLED: Visibility labels feature is not available") \ + unless exists?(VisibilityConstants::LABELS_TABLE_NAME) + end + + # Does table exist? + def exists?(table_name) + @admin.tableExists(table_name) + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb new file mode 100644 index 0000000000..b32e691cf8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/irb/hirb.rb @@ -0,0 +1,59 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +require 'rbconfig' + +module IRB + WINDOZE = Config::CONFIG['host_os'] =~ /mswin|mingw/ + + # Subclass of IRB so can intercept methods + class HIRB < Irb + def initialize + # This is ugly. Our 'help' method above provokes the following message + # on irb construction: 'irb: warn: can't alias help from irb_help.' + # Below, we reset the output so its pointed at /dev/null during irb + # construction just so this message does not come out after we emit + # the banner. Other attempts at playing with the hash of methods + # down in IRB didn't seem to work. I think the worst thing that can + # happen is the shell exiting because of failed IRB construction with + # no error (though we're not blanking STDERR) + begin + # Map the '/dev/null' according to the runing platform + # Under Windows platform the 'dev/null' is not fully compliant with unix, + # and the 'NUL' object need to be use instead. + devnull = "/dev/null" + devnull = "NUL" if WINDOZE + f = File.open(devnull, "w") + $stdout = f + super + ensure + f.close() + $stdout = STDOUT + end + end + + def output_value + # Suppress output if last_value is 'nil' + # Otherwise, when user types help, get ugly 'nil' + # after all output. + if @context.last_value != nil + super + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb new file mode 100644 index 0000000000..994a195d27 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell.rb @@ -0,0 +1,395 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Shell commands module +module Shell + @@commands = {} + def self.commands + @@commands + end + + @@command_groups = {} + def self.command_groups + @@command_groups + end + + def self.load_command(name, group, aliases=[]) + return if commands[name] + + # Register command in the group + raise ArgumentError, "Unknown group: #{group}" unless command_groups[group] + command_groups[group][:commands] << name + + # Load command + begin + require "shell/commands/#{name}" + klass_name = name.to_s.gsub(/(?:^|_)(.)/) { $1.upcase } # camelize + commands[name] = eval("Commands::#{klass_name}") + aliases.each do |an_alias| + commands[an_alias] = commands[name] + end + rescue => e + raise "Can't load hbase shell command: #{name}. Error: #{e}\n#{e.backtrace.join("\n")}" + end + end + + def self.load_command_group(group, opts) + raise ArgumentError, "No :commands for group #{group}" unless opts[:commands] + + command_groups[group] = { + :commands => [], + :command_names => opts[:commands], + :full_name => opts[:full_name] || group, + :comment => opts[:comment] + } + + all_aliases = opts[:aliases] || {} + + opts[:commands].each do |command| + aliases = all_aliases[command] || [] + load_command(command, group, aliases) + end + end + + #---------------------------------------------------------------------- + class Shell + attr_accessor :hbase + attr_accessor :formatter + + @debug = false + attr_accessor :debug + + def initialize(hbase, formatter) + self.hbase = hbase + self.formatter = formatter + end + + def hbase_admin + @hbase_admin ||= hbase.admin(formatter) + end + + def hbase_table(name) + hbase.table(name, self) + end + + def hbase_replication_admin + @hbase_replication_admin ||= hbase.replication_admin(formatter) + end + + def hbase_security_admin + @hbase_security_admin ||= hbase.security_admin(formatter) + end + + def hbase_visibility_labels_admin + @hbase_visibility_labels_admin ||= hbase.visibility_labels_admin(formatter) + end + + def export_commands(where) + ::Shell.commands.keys.each do |cmd| + # here where is the IRB namespace + # this method just adds the call to the specified command + # which just references back to 'this' shell object + # a decently extensible way to add commands + where.send :instance_eval, <<-EOF + def #{cmd}(*args) + ret = @shell.command('#{cmd}', *args) + puts + return ret + end + EOF + end + end + + def command_instance(command) + ::Shell.commands[command.to_s].new(self) + end + + #call the method 'command' on the specified command + def command(command, *args) + internal_command(command, :command, *args) + end + + #call a specific internal method in the command instance + # command - name of the command to call + # method_name - name of the method on the command to call. Defaults to just 'command' + # args - to be passed to the named method + def internal_command(command, method_name= :command, *args) + command_instance(command).command_safe(self.debug,method_name, *args) + end + + def print_banner + puts "HBase Shell; enter 'help' for list of supported commands." + puts 'Type "exit" to leave the HBase Shell' + print 'Version ' + command('version') + puts + end + + def help_multi_command(command) + puts "Command: #{command}" + puts command_instance(command).help + puts + return nil + end + + def help_command(command) + puts command_instance(command).help + return nil + end + + def help_group(group_name) + group = ::Shell.command_groups[group_name.to_s] + group[:commands].sort.each { |cmd| help_multi_command(cmd) } + if group[:comment] + puts '-' * 80 + puts + puts group[:comment] + puts + end + return nil + end + + def help(command = nil) + if command + return help_command(command) if ::Shell.commands[command.to_s] + return help_group(command) if ::Shell.command_groups[command.to_s] + puts "ERROR: Invalid command or command group name: #{command}" + puts + end + + puts help_header + puts + puts 'COMMAND GROUPS:' + ::Shell.command_groups.each do |name, group| + puts " Group name: " + name + puts " Commands: " + group[:command_names].sort.join(', ') + puts + end + unless command + puts 'SHELL USAGE:' + help_footer + end + return nil + end + + def help_header + return "HBase Shell, version #{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + "\n" + + "Type 'help \"COMMAND\"', (e.g. 'help \"get\"' -- the quotes are necessary) for help on a specific command.\n" + + "Commands are grouped. Type 'help \"COMMAND_GROUP\"', (e.g. 'help \"general\"') for help on a command group." + end + + def help_footer + puts <<-HERE +Quote all names in HBase Shell such as table and column names. Commas delimit +command parameters. Type after entering a command to run it. +Dictionaries of configuration used in the creation and alteration of tables are +Ruby Hashes. They look like this: + + {'key1' => 'value1', 'key2' => 'value2', ...} + +and are opened and closed with curley-braces. Key/values are delimited by the +'=>' character combination. Usually keys are predefined constants such as +NAME, VERSIONS, COMPRESSION, etc. Constants do not need to be quoted. Type +'Object.constants' to see a (messy) list of all constants in the environment. + +If you are using binary keys or values and need to enter them in the shell, use +double-quote'd hexadecimal representation. For example: + + hbase> get 't1', "key\\x03\\x3f\\xcd" + hbase> get 't1', "key\\003\\023\\011" + hbase> put 't1', "test\\xef\\xff", 'f1:', "\\x01\\x33\\x40" + +The HBase shell is the (J)Ruby IRB with the above HBase-specific commands added. +For more on the HBase Shell, see http://hbase.apache.org/book.html + HERE + end + end +end + +# Load commands base class +require 'shell/commands' + +# Load all commands +Shell.load_command_group( + 'general', + :full_name => 'GENERAL HBASE SHELL COMMANDS', + :commands => %w[ + status + version + table_help + whoami + ] +) + +Shell.load_command_group( + 'ddl', + :full_name => 'TABLES MANAGEMENT COMMANDS', + :commands => %w[ + alter + create + describe + disable + disable_all + is_disabled + drop + drop_all + enable + enable_all + is_enabled + exists + list + show_filters + alter_status + alter_async + get_table + ], + :aliases => { + 'describe' => ['desc'] + } +) + +Shell.load_command_group( + 'namespace', + :full_name => 'NAMESPACE MANAGEMENT COMMANDS', + :commands => %w[ + create_namespace + drop_namespace + alter_namespace + describe_namespace + list_namespace + list_namespace_tables + ] +) + +Shell.load_command_group( + 'dml', + :full_name => 'DATA MANIPULATION COMMANDS', + :commands => %w[ + count + delete + deleteall + get + get_counter + incr + put + scan + truncate + truncate_preserve + append + ] +) + +Shell.load_command_group( + 'tools', + :full_name => 'HBASE SURGERY TOOLS', + :comment => "WARNING: Above commands are for 'experts'-only as misuse can damage an install", + :commands => %w[ + assign + balancer + balance_switch + close_region + compact + flush + major_compact + move + split + merge_region + unassign + zk_dump + wal_roll + catalogjanitor_run + catalogjanitor_switch + catalogjanitor_enabled + compact_rs + trace + ], + # TODO remove older hlog_roll command + :aliases => { + 'wal_roll' => ['hlog_roll'] + } +) + +Shell.load_command_group( + 'replication', + :full_name => 'CLUSTER REPLICATION TOOLS', + :comment => "In order to use these tools, hbase.replication must be true.", + :commands => %w[ + add_peer + remove_peer + list_peers + enable_peer + disable_peer + show_peer_tableCFs + set_peer_tableCFs + list_replicated_tables + append_peer_tableCFs + remove_peer_tableCFs + enable_table_replication + disable_table_replication + ] +) + +Shell.load_command_group( + 'snapshots', + :full_name => 'CLUSTER SNAPSHOT TOOLS', + :commands => %w[ + snapshot + clone_snapshot + restore_snapshot + delete_snapshot + delete_all_snapshot + list_snapshots + ] +) + +Shell.load_command_group( + 'configuration', + :full_name => 'ONLINE CONFIGURATION TOOLS', + :commands => %w[ + update_config + update_all_config + ] +) + +Shell.load_command_group( + 'security', + :full_name => 'SECURITY TOOLS', + :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", + :commands => %w[ + grant + revoke + user_permission + ] +) + +Shell.load_command_group( + 'visibility labels', + :full_name => 'VISIBILITY LABEL TOOLS', + :comment => "NOTE: Above commands are only applicable if running with the VisibilityController coprocessor", + :commands => %w[ + add_labels + list_labels + set_auths + get_auths + clear_auths + set_visibility + ] +) diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb new file mode 100644 index 0000000000..54fa204048 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands.rb @@ -0,0 +1,138 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Command + + def initialize(shell) + @shell = shell + end + + #wrap an execution of cmd to catch hbase exceptions + # cmd - command name to execture + # args - arguments to pass to the command + def command_safe(debug, cmd = :command, *args) + # send is internal ruby method to call 'cmd' with *args + #(everything is a message, so this is just the formal semantics to support that idiom) + translate_hbase_exceptions(*args) { send(cmd,*args) } + rescue => e + rootCause = e + while rootCause != nil && rootCause.respond_to?(:cause) && rootCause.cause != nil + rootCause = rootCause.cause + end + puts + puts "ERROR: #{rootCause}" + puts "Backtrace: #{rootCause.backtrace.join("\n ")}" if debug + puts + puts "Here is some help for this command:" + puts help + puts + end + + def admin + @shell.hbase_admin + end + + def table(name) + @shell.hbase_table(name) + end + + def replication_admin + @shell.hbase_replication_admin + end + + def security_admin + @shell.hbase_security_admin + end + + def visibility_labels_admin + @shell.hbase_visibility_labels_admin + end + + #---------------------------------------------------------------------- + + def formatter + @shell.formatter + end + + def format_simple_command + now = Time.now + yield + formatter.header + formatter.footer(now) + end + + def format_and_return_simple_command + now = Time.now + ret = yield + formatter.header + formatter.footer(now) + return ret + end + + def translate_hbase_exceptions(*args) + yield + rescue => e + raise e unless e.respond_to?(:cause) && e.cause != nil + + # Get the special java exception which will be handled + cause = e.cause + if cause.kind_of?(org.apache.hadoop.hbase.TableNotFoundException) then + str = java.lang.String.new("#{cause}") + raise "Unknown table #{str}!" + end + if cause.kind_of?(org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException) then + exceptions = cause.getCauses + exceptions.each do |exception| + if exception.kind_of?(org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException) then + valid_cols = table(args.first).get_all_columns.map { |c| c + '*' } + raise "Unknown column family! Valid column names: #{valid_cols.join(", ")}" + end + end + end + if cause.kind_of?(org.apache.hadoop.hbase.TableExistsException) then + str = java.lang.String.new("#{cause}") + strs = str.split("\n") + if strs.size > 0 then + s = strs[0].split(' '); + if(s.size > 1) + raise "Table already exists: #{s[1]}!" + end + raise "Table already exists: #{strs[0]}!" + end + end + # To be safe, here only AccessDeniedException is considered. In future + # we might support more in more generic approach when possible. + if cause.kind_of?(org.apache.hadoop.hbase.security.AccessDeniedException) then + str = java.lang.String.new("#{cause}") + # Error message is merged with stack trace, reference StringUtils.stringifyException + # This is to parse and get the error message from the whole. + strs = str.split("\n") + if strs.size > 0 then + raise "#{strs[0]}" + end + end + + # Throw the other exception which hasn't been handled above + raise e + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb new file mode 100644 index 0000000000..65a1140f24 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_labels.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddLabels < Command + def help + return <<-EOF +Add a set of visibility labels. +Syntax : add_labels [label1, label2] + +For example: + + hbase> add_labels ['SECRET','PRIVATE'] +EOF + end + + def command(*args) + format_simple_command do + visibility_labels_admin.add_labels(args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb new file mode 100644 index 0000000000..be01041644 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/add_peer.rb @@ -0,0 +1,70 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AddPeer< Command + def help + return <<-EOF +A peer can either be another HBase cluster or a custom replication endpoint. In either case an id +must be specified to identify the peer. + +For a HBase cluster peer, a cluster key must be provided and is composed like this: +hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent +This gives a full path for HBase to connect to another HBase cluster. An optional parameter for +table column families identifies which column families will be replicated to the peer cluster. +Examples: + + hbase> add_peer '1', "server1.cie.com:2181:/hbase" + hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" + hbase> add_peer '3', "zk4,zk5,zk6:11000:/hbase-test", "table1; table2:cf1; table3:cf1,cf2" + hbase> add_peer '4', CLUSTER_KEY => "server1.cie.com:2181:/hbase" + hbase> add_peer '5', CLUSTER_KEY => "server1.cie.com:2181:/hbase", + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + +For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments +are DATA and CONFIG which can be specified to set different either the peer_data or configuration +for the custom replication endpoint. Table column families is optional and can be specified with +the key TABLE_CFS. + + hbase> add_peer '6', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint' + hbase> add_peer '7', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 } + hbase> add_peer '8', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + CONFIG => { "config1" => "value1", "config2" => "value2" } + hbase> add_peer '9', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + hbase> add_peer '10', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + hbase> add_peer '11', ENDPOINT_CLASSNAME => 'org.apache.hadoop.hbase.MyReplicationEndpoint', + DATA => { "key1" => 1 }, CONFIG => { "config1" => "value1", "config2" => "value2" }, + TABLE_CFS => { "table1" => [], "table2" => ["cf1"], "table3" => ["cf1", "cf2"] } + +Note: Either CLUSTER_KEY or ENDPOINT_CLASSNAME must be specified but not both. +EOF + end + + def command(id, args = {}, peer_tableCFs = nil) + format_simple_command do + replication_admin.add_peer(id, args, peer_tableCFs) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb new file mode 100644 index 0000000000..2c3aa6f872 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter.rb @@ -0,0 +1,101 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Alter < Command + def help + return <<-EOF +Alter a table. If the "hbase.online.schema.update.enable" property is set to +false, then the table must be disabled (see help 'disable'). If the +"hbase.online.schema.update.enable" property is set to true, tables can be +altered without disabling them first. Altering enabled tables has caused problems +in the past, so use caution and test it before using in production. + +You can use the alter command to add, +modify or delete column families or change table configuration options. +Column families work in a similar way as the 'create' command. The column family +specification can either be a name string, or a dictionary with the NAME attribute. +Dictionaries are described in the output of the 'help' command, with no arguments. + +For example, to change or add the 'f1' column family in table 't1' from +current value to keep a maximum of 5 cell VERSIONS, do: + + hbase> alter 't1', NAME => 'f1', VERSIONS => 5 + +You can operate on several column families: + + hbase> alter 't1', 'f1', {NAME => 'f2', IN_MEMORY => true}, {NAME => 'f3', VERSIONS => 5} + +To delete the 'f1' column family in table 'ns1:t1', use one of: + + hbase> alter 'ns1:t1', NAME => 'f1', METHOD => 'delete' + hbase> alter 'ns1:t1', 'delete' => 'f1' + +You can also change table-scope attributes like MAX_FILESIZE, READONLY, +MEMSTORE_FLUSHSIZE, DURABILITY, etc. These can be put at the end; +for example, to change the max size of a region to 128MB, do: + + hbase> alter 't1', MAX_FILESIZE => '134217728' + +You can add a table coprocessor by setting a table coprocessor attribute: + + hbase> alter 't1', + 'coprocessor'=>'hdfs:///foo.jar|com.foo.FooRegionObserver|1001|arg1=1,arg2=2' + +Since you can have multiple coprocessors configured for a table, a +sequence number will be automatically appended to the attribute name +to uniquely identify it. + +The coprocessor attribute must match the pattern below in order for +the framework to understand how to load the coprocessor classes: + + [coprocessor jar file location] | class name | [priority] | [arguments] + +You can also set configuration settings specific to this table or column family: + + hbase> alter 't1', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'} + hbase> alter 't1', {NAME => 'f2', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} + +You can also remove a table-scope attribute: + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'MAX_FILESIZE' + + hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' + +You can also set REGION_REPLICATION: + + hbase> alter 't1', {REGION_REPLICATION => 2} + +There could be more than one alteration in one command: + + hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, + { MAX_FILESIZE => '134217728' }, { METHOD => 'delete', NAME => 'f2' }, + OWNER => 'johndoe', METADATA => { 'mykey' => 'myvalue' } +EOF + end + + def command(table, *args) + format_simple_command do + admin.alter(table, true, *args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb new file mode 100644 index 0000000000..bddff010a2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_async.rb @@ -0,0 +1,65 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterAsync < Command + def help + return <<-EOF +Alter column family schema, does not wait for all regions to receive the +schema changes. Pass table name and a dictionary specifying new column +family schema. Dictionaries are described on the main help command output. +Dictionary must include name of column family to alter. For example, + +To change or add the 'f1' column family in table 't1' from defaults +to instead keep a maximum of 5 cell VERSIONS, do: + + hbase> alter_async 't1', NAME => 'f1', VERSIONS => 5 + +To delete the 'f1' column family in table 'ns1:t1', do: + + hbase> alter_async 'ns1:t1', NAME => 'f1', METHOD => 'delete' + +or a shorter version: + + hbase> alter_async 'ns1:t1', 'delete' => 'f1' + +You can also change table-scope attributes like MAX_FILESIZE +MEMSTORE_FLUSHSIZE, READONLY, and DEFERRED_LOG_FLUSH. + +For example, to change the max size of a family to 128MB, do: + + hbase> alter 't1', METHOD => 'table_att', MAX_FILESIZE => '134217728' + +There could be more than one alteration in one command: + + hbase> alter 't1', {NAME => 'f1'}, {NAME => 'f2', METHOD => 'delete'} + +To check if all the regions have been updated, use alter_status +EOF + end + + def command(table, *args) + format_simple_command do + admin.alter(table, false, *args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb new file mode 100644 index 0000000000..760bbf7bc5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_namespace.rb @@ -0,0 +1,44 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterNamespace < Command + def help + return <<-EOF +Alter namespace properties. + +To add/modify a property: + + hbase> alter_namespace 'ns1', {METHOD => 'set', 'PROERTY_NAME' => 'PROPERTY_VALUE'} + +To delete a property: + + hbase> alter_namespace 'ns1', {METHOD => 'unset', NAME=>'PROERTY_NAME'} +EOF + end + + def command(namespace, *args) + format_simple_command do + admin.alter_namespace(namespace, *args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb new file mode 100644 index 0000000000..72f044dbba --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/alter_status.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AlterStatus < Command + def help + return <<-EOF +Get the status of the alter command. Indicates the number of regions of the +table that have received the updated schema +Pass table name. + +hbase> alter_status 't1' +hbase> alter_status 'ns1:t1' +EOF + end + def command(table) + admin.alter_status(table) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb new file mode 100644 index 0000000000..a0ef36d28e --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append.rb @@ -0,0 +1,52 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Append < Command + def help + return <<-EOF +Appends a cell 'value' at specified table/row/column coordinates. + + hbase> append 't1', 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> append 't1', 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.append 'r1', 'c1', 'value', ATTRIBUTES=>{'mykey'=>'myvalue'} + hbase> t.append 'r1', 'c1', 'value', {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, value, args={}) + append(table(table), row, column, value, args) + end + + def append(table, row, column, value, args={}) + format_simple_command do + table._append_internal(row, column, value, args) + end + end + end + end +end + +#add incr comamnd to Table +::Hbase::Table.add_shell_command("append") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb new file mode 100644 index 0000000000..3919b20317 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/append_peer_tableCFs.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class AppendPeerTableCFs< Command + def help + return <<-EOF +Append a replicable table-cf config for the specified peer +Examples: + + # append a table / table-cf to be replicable for a peer + hbase> append_peer_tableCFs '2', "table4:cfA,cfB" + +EOF + end + + def command(id, table_cfs) + format_simple_command do + replication_admin.append_peer_tableCFs(id, table_cfs) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb new file mode 100644 index 0000000000..448a5460b4 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/assign.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Assign < Command + def help + return <<-EOF +Assign a region. Use with caution. If region already assigned, +this command will do a force reassign. For experts only. +Examples: + + hbase> assign 'REGIONNAME' + hbase> assign 'ENCODED_REGIONNAME' +EOF + end + + def command(region_name) + format_simple_command do + admin.assign(region_name) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb new file mode 100644 index 0000000000..4d7778daa4 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balance_switch.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class BalanceSwitch < Command + def help + return <<-EOF +Enable/Disable balancer. Returns previous balancer state. +Examples: + + hbase> balance_switch true + hbase> balance_switch false +EOF + end + + def command(enableDisable) + format_simple_command do + formatter.row([ + admin.balance_switch(enableDisable)? "true" : "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb new file mode 100644 index 0000000000..c329eced4b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/balancer.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Balancer < Command + def help + return <<-EOF +Trigger the cluster balancer. Returns true if balancer ran and was able to +tell the region servers to unassign all the regions to balance (the re-assignment itself is async). +Otherwise false (Will not run if regions in transition). +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.balancer()? "true": "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb new file mode 100644 index 0000000000..b310c3a7ab --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_enabled.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorEnabled < Command + def help + return <<-EOF +Query for the CatalogJanitor state (enabled/disabled?) +Examples: + + hbase> catalogjanitor_enabled +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.catalogjanitor_enabled()? "true" : "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb new file mode 100644 index 0000000000..03426cb4f1 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_run.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorRun < Command + def help + return <<-EOF +Catalog janitor command to run the (garbage collection) scan from command line. + + hbase> catalogjanitor_run + +EOF + end + def command() + format_simple_command do + admin.catalogjanitor_run() + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb new file mode 100644 index 0000000000..fce1925eec --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/catalogjanitor_switch.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CatalogjanitorSwitch < Command + def help + return <<-EOF +Enable/Disable CatalogJanitor. Returns previous CatalogJanitor state. +Examples: + + hbase> catalogjanitor_switch true + hbase> catalogjanitor_switch false +EOF + end + + def command(enableDisable) + format_simple_command do + formatter.row([ + admin.catalogjanitor_switch(enableDisable)? "true" : "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb new file mode 100644 index 0000000000..8553fa6471 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clear_auths.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearAuths < Command + def help + return <<-EOF +Clear visibility labels from a user or group +Syntax : clear_auths 'user',[label1, label2] + +For example: + + hbase> clear_auths 'user1', ['SECRET','PRIVATE'] + hbase> clear_auths '@group1', ['SECRET','PRIVATE'] +EOF + end + + def command(user, *args) + format_simple_command do + visibility_labels_admin.clear_auths(user, args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb new file mode 100644 index 0000000000..8c193bb7c1 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/clone_snapshot.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloneSnapshot < Command + def help + return <<-EOF +Create a new table by cloning the snapshot content. +There're no copies of data involved. +And writing on the newly created table will not influence the snapshot data. + +Examples: + hbase> clone_snapshot 'snapshotName', 'tableName' + hbase> clone_snapshot 'snapshotName', 'namespace:tableName' +EOF + end + + def command(snapshot_name, table) + format_simple_command do + admin.clone_snapshot(snapshot_name, table) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb new file mode 100644 index 0000000000..9e2900c31b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/close_region.rb @@ -0,0 +1,61 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CloseRegion < Command + def help + return <<-EOF +Close a single region. Ask the master to close a region out on the cluster +or if 'SERVER_NAME' is supplied, ask the designated hosting regionserver to +close the region directly. Closing a region, the master expects 'REGIONNAME' +to be a fully qualified region name. When asking the hosting regionserver to +directly close a region, you pass the regions' encoded name only. A region +name looks like this: + + TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. +or + Namespace:TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. + +The trailing period is part of the regionserver name. A region's encoded name +is the hash at the end of a region name; e.g. 527db22f95c8a9e0116f0cc13c680396 +(without the period). A 'SERVER_NAME' is its host, port plus startcode. For +example: host187.example.com,60020,1289493121758 (find servername in master ui +or when you do detailed status in shell). This command will end up running +close on the region hosting regionserver. The close is done without the +master's involvement (It will not know of the close). Once closed, region will +stay closed. Use assign to reopen/reassign. Use unassign or move to assign +the region elsewhere on cluster. Use with caution. For experts only. +Examples: + + hbase> close_region 'REGIONNAME' + hbase> close_region 'REGIONNAME', 'SERVER_NAME' + hbase> close_region 'ENCODED_REGIONNAME' + hbase> close_region 'ENCODED_REGIONNAME', 'SERVER_NAME' +EOF + end + + def command(region_name, server = nil) + format_simple_command do + admin.close_region(region_name, server) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb new file mode 100644 index 0000000000..adeffa2d03 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact.rb @@ -0,0 +1,48 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Compact < Command + def help + return <<-EOF + Compact all regions in passed table or pass a region row + to compact an individual region. You can also compact a single column + family within a region. + Examples: + Compact all regions in a table: + hbase> compact 'ns1:t1' + hbase> compact 't1' + Compact an entire region: + hbase> compact 'r1' + Compact only a column family within a region: + hbase> compact 'r1', 'c1' + Compact a column family within a table: + hbase> compact 't1', 'c1' + EOF + end + + def command(table_or_region_name, family = nil) + format_simple_command do + admin.compact(table_or_region_name, family) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb new file mode 100644 index 0000000000..0ecdd21113 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/compact_rs.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CompactRs < Command + def help + return <<-EOF + Compact all regions on passed regionserver. + Examples: + Compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020' + or + hbase> compact_rs 'host187.example.com,60020,1289493121758' + Major compact all regions on a regionserver: + hbase> compact_rs 'host187.example.com,60020,1289493121758', true + EOF + end + + def command(regionserver, major = false) + format_simple_command do + admin.compact_regionserver(regionserver, major) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb new file mode 100644 index 0000000000..225005ee36 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/count.rb @@ -0,0 +1,77 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Count < Command + def help + return <<-EOF +Count the number of rows in a table. Return value is the number of rows. +This operation may take a LONG time (Run '$HADOOP_HOME/bin/hadoop jar +hbase.jar rowcount' to run a counting mapreduce job). Current count is shown +every 1000 rows by default. Count interval may be optionally specified. Scan +caching is enabled on count scans by default. Default cache size is 10 rows. +If your rows are small in size, you may want to increase this +parameter. Examples: + + hbase> count 'ns1:t1' + hbase> count 't1' + hbase> count 't1', INTERVAL => 100000 + hbase> count 't1', CACHE => 1000 + hbase> count 't1', INTERVAL => 10, CACHE => 1000 + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding commands would be: + + hbase> t.count + hbase> t.count INTERVAL => 100000 + hbase> t.count CACHE => 1000 + hbase> t.count INTERVAL => 10, CACHE => 1000 +EOF + end + + def command(table, params = {}) + count(table(table), params) + end + + def count(table, params = {}) + # If the second parameter is an integer, then it is the old command syntax + params = { 'INTERVAL' => params } if params.kind_of?(Fixnum) + + # Merge params with defaults + params = { + 'INTERVAL' => 1000, + 'CACHE' => 10 + }.merge(params) + + # Call the counter method + now = Time.now + formatter.header + count = table._count_internal(params['INTERVAL'].to_i, params['CACHE'].to_i) do |cnt, row| + formatter.row([ "Current count: #{cnt}, row: #{row}" ]) + end + formatter.footer(now, count) + return count + end + end + end +end + +#Add the method table.count that calls count.count +::Hbase::Table.add_shell_command("count") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb new file mode 100644 index 0000000000..ab3a3d1069 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create.rb @@ -0,0 +1,72 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Create < Command + def help + return <<-EOF +Creates a table. Pass a table name, and a set of column family +specifications (at least one), and, optionally, table configuration. +Column specification can be a simple string (name), or a dictionary +(dictionaries are described below in main help output), necessarily +including NAME attribute. +Examples: + +Create a table with namespace=ns1 and table qualifier=t1 + hbase> create 'ns1:t1', {NAME => 'f1', VERSIONS => 5} + +Create a table with namespace=default and table qualifier=t1 + hbase> create 't1', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'} + hbase> # The above in shorthand would be the following: + hbase> create 't1', 'f1', 'f2', 'f3' + hbase> create 't1', {NAME => 'f1', VERSIONS => 1, TTL => 2592000, BLOCKCACHE => true} + hbase> create 't1', {NAME => 'f1', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} + +Table configuration options can be put at the end. +Examples: + + hbase> create 'ns1:t1', 'f1', SPLITS => ['10', '20', '30', '40'] + hbase> create 't1', 'f1', SPLITS => ['10', '20', '30', '40'] + hbase> create 't1', 'f1', SPLITS_FILE => 'splits.txt', OWNER => 'johndoe' + hbase> create 't1', {NAME => 'f1', VERSIONS => 5}, METADATA => { 'mykey' => 'myvalue' } + hbase> # Optionally pre-split the table into NUMREGIONS, using + hbase> # SPLITALGO ("HexStringSplit", "UniformSplit" or classname) + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit'} + hbase> create 't1', 'f1', {NUMREGIONS => 15, SPLITALGO => 'HexStringSplit', REGION_REPLICATION => 2, CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'}} + +You can also keep around a reference to the created table: + + hbase> t1 = create 't1', 'f1' + +Which gives you a reference to the table named 't1', on which you can then +call methods. +EOF + end + + def command(table, *args) + format_simple_command do + ret = admin.create(table, *args) + end + #and then return the table you just created + table(table) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb new file mode 100644 index 0000000000..3259eb6b11 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/create_namespace.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CreateNamespace < Command + def help + return <<-EOF +Create namespace; pass namespace name, +and optionally a dictionary of namespace configuration. +Examples: + + hbase> create_namespace 'ns1' + hbase> create_namespace 'ns1', {'PROERTY_NAME'=>'PROPERTY_VALUE'} +EOF + end + + def command(namespace, *args) + format_simple_command do + admin.create_namespace(namespace, *args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb new file mode 100644 index 0000000000..dcb83414f6 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete.rb @@ -0,0 +1,59 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Delete < Command + def help + return <<-EOF +Put a delete cell value at specified table/row/column and optionally +timestamp coordinates. Deletes must match the deleted cell's +coordinates exactly. When scanning, a delete cell suppresses older +versions. To delete a cell from 't1' at row 'r1' under column 'c1' +marked with the time 'ts1', do: + + hbase> delete 'ns1:t1', 'r1', 'c1', ts1 + hbase> delete 't1', 'r1', 'c1', ts1 + hbase> delete 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +The same command can also be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.delete 'r1', 'c1', ts1 + hbase> t.delete 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + delete(table(table), row, column, timestamp, args) + end + + def delete(table, row, column, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + format_simple_command do + table._delete_internal(row, column, timestamp, args) + end + end + end + end +end + +#Add the method table.delete that calls delete.delete +::Hbase::Table.add_shell_command("delete") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb new file mode 100644 index 0000000000..bc072597bb --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_all_snapshot.rb @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteAllSnapshot < Command + def help + return <<-EOF +Delete all of the snapshots matching the given regex. Examples: + + hbase> delete_all_snapshot 's.*' + +EOF + end + + def command(regex) + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + list = admin.list_snapshot(regex) + count = list.size + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + puts "\nDelete the above #{count} snapshots (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No snapshots matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + format_simple_command do + admin.delete_all_snapshot(regex) + end + list = admin.list_snapshot(regex) + leftOverSnapshotCount = list.size + successfullyDeleted = count - leftOverSnapshotCount + puts "#{successfullyDeleted} snapshots successfully deleted." unless successfullyDeleted == 0 + return if leftOverSnapshotCount == 0 + puts "\nFailed to delete the below #{leftOverSnapshotCount} snapshots." + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb new file mode 100644 index 0000000000..b8c3791a54 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/delete_snapshot.rb @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DeleteSnapshot < Command + def help + return <<-EOF +Delete a specified snapshot. Examples: + + hbase> delete_snapshot 'snapshotName', +EOF + end + + def command(snapshot_name) + format_simple_command do + admin.delete_snapshot(snapshot_name) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb new file mode 100644 index 0000000000..e6118c9144 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/deleteall.rb @@ -0,0 +1,60 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Deleteall < Command + def help + return <<-EOF +Delete all cells in a given row; pass a table name, row, and optionally +a column and timestamp. Examples: + + hbase> deleteall 'ns1:t1', 'r1' + hbase> deleteall 't1', 'r1' + hbase> deleteall 't1', 'r1', 'c1' + hbase> deleteall 't1', 'r1', 'c1', ts1 + hbase> deleteall 't1', 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.deleteall 'r1' + hbase> t.deleteall 'r1', 'c1' + hbase> t.deleteall 'r1', 'c1', ts1 + hbase> t.deleteall 'r1', 'c1', ts1, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + deleteall(table(table), row, column, timestamp, args) + end + + def deleteall(table, row, column = nil, + timestamp = org.apache.hadoop.hbase.HConstants::LATEST_TIMESTAMP, args = {}) + format_simple_command do + table._deleteall_internal(row, column, timestamp, args) + end + end + end + end +end + +#Add the method table.deleteall that calls deleteall.deleteall +::Hbase::Table.add_shell_command("deleteall") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb new file mode 100644 index 0000000000..bfa16cdc2d --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Describe < Command + def help + return <<-EOF +Describe the named table. For example: + hbase> describe 't1' + hbase> describe 'ns1:t1' + +Alternatively, you can use the abbreviated 'desc' for the same thing. + hbase> desc 't1' + hbase> desc 'ns1:t1' +EOF + end + + def command(table) + now = Time.now + + column_families = admin.get_column_families(table) + + formatter.header(["Table " + table.to_s + " is " + if admin.enabled?(table) then "ENABLED" else "DISABLED" end]) + formatter.row([table.to_s + admin.get_table_attributes(table)], true) + formatter.header(["COLUMN FAMILIES DESCRIPTION"]) + column_families.each do |column_family| + formatter.row([ column_family.to_s ], true) + end + formatter.footer(now) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb new file mode 100644 index 0000000000..cf135da558 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/describe_namespace.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DescribeNamespace < Command + def help + return <<-EOF +Describe the named namespace. For example: + hbase> describe_namespace 'ns1' +EOF + end + + def command(namespace) + now = Time.now + + desc = admin.describe_namespace(namespace) + + formatter.header([ "DESCRIPTION" ], [ 64 ]) + formatter.row([ desc ], true, [ 64 ]) + formatter.footer(now) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb new file mode 100644 index 0000000000..79bcd864f7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Disable < Command + def help + return <<-EOF +Start disable of named table: + hbase> disable 't1' + hbase> disable 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + admin.disable(table) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb new file mode 100644 index 0000000000..212db249a8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableAll < Command + def help + return <<-EOF +Disable all of tables matching the given regex: + +hbase> disable_all 't.*' +hbase> disable_all 'ns:t.*' +hbase> disable_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nDisable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.disable_all(regex) + puts "#{count - failed.size} tables successfully disabled" + puts "#{failed.size} tables not disabled due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb new file mode 100644 index 0000000000..416545b21f --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_peer.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisablePeer< Command + def help + return <<-EOF +Stops the replication stream to the specified cluster, but still +keeps track of new edits to replicate. + +Examples: + + hbase> disable_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.disable_peer(id) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb new file mode 100644 index 0000000000..5bf9667949 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/disable_table_replication.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableTableReplication< Command + def help + return <<-EOF +Disable a table's replication switch. + +Examples: + + hbase> disable_table_replication 'table_name' +EOF + end + + def command(table_name) + format_simple_command do + replication_admin.disable_tablerep(table_name) + end + puts "The replication swith of table '#{table_name}' successfully disabled" + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb new file mode 100644 index 0000000000..fc7b134468 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Drop < Command + def help + return <<-EOF +Drop the named table. Table must first be disabled: + hbase> drop 't1' + hbase> drop 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + admin.drop(table) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb new file mode 100644 index 0000000000..73398fbf6e --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DropAll < Command + def help + return <<-EOF +Drop all of the tables matching the given regex: + +hbase> drop_all 't.*' +hbase> drop_all 'ns:t.*' +hbase> drop_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nDrop the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.drop_all(regex) + puts "#{count - failed.size} tables successfully dropped" + puts "#{failed.size} tables not dropped due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb new file mode 100644 index 0000000000..b030d279c7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/drop_namespace.rb @@ -0,0 +1,36 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DropNamespace < Command + def help + return <<-EOF +Drop the named namespace. The namespace must be empty. +EOF + end + + def command(namespace) + format_simple_command do + admin.drop_namespace(namespace) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb new file mode 100644 index 0000000000..deeb70cce2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Enable < Command + def help + return <<-EOF +Start enable of named table: + hbase> enable 't1' + hbase> enable 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + admin.enable(table) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb new file mode 100644 index 0000000000..2b899f2ac0 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_all.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableAll < Command + def help + return <<-EOF +Enable all of the tables matching the given regex: + +hbase> enable_all 't.*' +hbase> enable_all 'ns:t.*' +hbase> enable_all 'ns:.*' +EOF + end + + def command(regex) + list = admin.list(regex) + count = list.size + list.each do |table| + formatter.row([ table ]) + end + puts "\nEnable the above #{count} tables (y/n)?" unless count == 0 + answer = 'n' + answer = gets.chomp unless count == 0 + puts "No tables matched the regex #{regex.to_s}" if count == 0 + return unless answer =~ /y.*/i + failed = admin.enable_all(regex) + puts "#{count - failed.size} tables successfully enabled" + puts "#{failed.size} tables not enabled due to an exception: #{failed.join ','}" unless failed.size == 0 + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb new file mode 100644 index 0000000000..55136ffb7f --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_peer.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnablePeer< Command + def help + return <<-EOF +Restarts the replication to the specified peer cluster, +continuing from where it was disabled. + +Examples: + + hbase> enable_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.enable_peer(id) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb new file mode 100644 index 0000000000..15e3133a89 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/enable_table_replication.rb @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableTableReplication< Command + def help + return <<-EOF +Enable a table's replication switch. + +Examples: + + hbase> enable_table_replication 'table_name' +EOF + end + + def command(table_name) + format_simple_command do + replication_admin.enable_tablerep(table_name) + end + puts "The replication swith of table '#{table_name}' successfully enabled" + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb new file mode 100644 index 0000000000..bacf6c94c4 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/exists.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Exists < Command + def help + return <<-EOF +Does the named table exist? + hbase> exists 't1' + hbase> exists 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + "Table #{table} " + (admin.exists?(table.to_s) ? "does exist" : "does not exist") + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb new file mode 100644 index 0000000000..2aefec52e9 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/flush.rb @@ -0,0 +1,41 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Flush < Command + def help + return <<-EOF +Flush all regions in passed table or pass a region row to +flush an individual region. For example: + + hbase> flush 'TABLENAME' + hbase> flush 'REGIONNAME' + hbase> flush 'ENCODED_REGIONNAME' +EOF + end + + def command(table_or_region_name) + format_simple_command do + admin.flush(table_or_region_name) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb new file mode 100644 index 0000000000..1ab13cb9ca --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get.rb @@ -0,0 +1,98 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Get < Command + def help + return <<-EOF +Get row or cell contents; pass table name, row, and optionally +a dictionary of column(s), timestamp, timerange and versions. Examples: + + hbase> get 'ns1:t1', 'r1' + hbase> get 't1', 'r1' + hbase> get 't1', 'r1', {TIMERANGE => [ts1, ts2]} + hbase> get 't1', 'r1', {COLUMN => 'c1'} + hbase> get 't1', 'r1', {COLUMN => ['c1', 'c2', 'c3']} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} + hbase> get 't1', 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} + hbase> get 't1', 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} + hbase> get 't1', 'r1', 'c1' + hbase> get 't1', 'r1', 'c1', 'c2' + hbase> get 't1', 'r1', ['c1', 'c2'] + hbase> get 't1', 'r1', {COLUMN => 'c1', ATTRIBUTES => {'mykey'=>'myvalue'}} + hbase> get 't1', 'r1', {COLUMN => 'c1', AUTHORIZATIONS => ['PRIVATE','SECRET']} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> get 't1', 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} + +Besides the default 'toStringBinary' format, 'get' also supports custom formatting by +column. A user can define a FORMATTER by adding it to the column name in the get +specification. The FORMATTER can be stipulated: + + 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) + 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. + +Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: + hbase> get 't1', 'r1' {COLUMN => ['cf:qualifier1:toInt', + 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } + +Note that you can specify a FORMATTER by column only (cf:qualifier). You cannot specify +a FORMATTER for all columns of a column family. + +The same commands also can be run on a reference to a table (obtained via get_table or +create_table). Suppose you had a reference t to table 't1', the corresponding commands +would be: + + hbase> t.get 'r1' + hbase> t.get 'r1', {TIMERANGE => [ts1, ts2]} + hbase> t.get 'r1', {COLUMN => 'c1'} + hbase> t.get 'r1', {COLUMN => ['c1', 'c2', 'c3']} + hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1} + hbase> t.get 'r1', {COLUMN => 'c1', TIMERANGE => [ts1, ts2], VERSIONS => 4} + hbase> t.get 'r1', {COLUMN => 'c1', TIMESTAMP => ts1, VERSIONS => 4} + hbase> t.get 'r1', {FILTER => "ValueFilter(=, 'binary:abc')"} + hbase> t.get 'r1', 'c1' + hbase> t.get 'r1', 'c1', 'c2' + hbase> t.get 'r1', ['c1', 'c2'] + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE'} + hbase> t.get 'r1', {CONSISTENCY => 'TIMELINE', REGION_REPLICA_ID => 1} +EOF + end + + def command(table, row, *args) + get(table(table), row, *args) + end + + def get(table, row, *args) + now = Time.now + formatter.header(["COLUMN", "CELL"]) + + table._get_internal(row, *args) do |column, value| + formatter.row([ column, value ]) + end + + formatter.footer(now) + end + end + end +end + +#add get command to table +::Hbase::Table.add_shell_command('get') diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb new file mode 100644 index 0000000000..1b758ef949 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_auths.rb @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetAuths < Command + def help + return <<-EOF +Get the visibility labels set for a particular user or group +Syntax : get_auths 'user' + +For example: + + hbase> get_auths 'user1' + hbase> get_auths '@group1' +EOF + end + + def command(user) + format_simple_command do + list = visibility_labels_admin.get_auths(user) + list.each do |auths| + formatter.row([org.apache.hadoop.hbase.util.Bytes::toStringBinary(auths.toByteArray)]) + end + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb new file mode 100644 index 0000000000..1264fe5f0f --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_counter.rb @@ -0,0 +1,54 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetCounter < Command + def help + return <<-EOF +Return a counter cell value at specified table/row/column coordinates. +A counter cell should be managed with atomic increment functions on HBase +and the data should be binary encoded (as long value). Example: + + hbase> get_counter 'ns1:t1', 'r1', 'c1' + hbase> get_counter 't1', 'r1', 'c1' + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.get_counter 'r1', 'c1' +EOF + end + + def command(table, row, column, dummy = nil) + get_counter(table(table), row, column) + end + + def get_counter(table, row, column, dummy = nil) + if cnt = table._get_counter_internal(row, column) + puts "COUNTER VALUE = #{cnt}" + else + puts "No counter found at specified coordinates" + end + end + end + end +end + +::Hbase::Table.add_shell_command('get_counter') diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb new file mode 100644 index 0000000000..43e7c1af70 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/get_table.rb @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class GetTable < Command + def help + return <<-EOF +Get the given table name and return it as an actual object to +be manipulated by the user. See table.help for more information +on how to use the table. +Eg. + + hbase> t1 = get_table 't1' + hbase> t1 = get_table 'ns1:t1' + +returns the table named 't1' as a table object. You can then do + + hbase> t1.help + +which will then print the help for that table. +EOF + end + + def command(table, *args) + format_and_return_simple_command do + table(table) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb new file mode 100644 index 0000000000..4a995ffe52 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/grant.rb @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Grant < Command + def help + return <<-EOF +Grant users specific rights. +Syntax : grant [<@namespace> [
[ []]] + +permissions is either zero or more letters from the set "RWXCA". +READ('R'), WRITE('W'), EXEC('X'), CREATE('C'), ADMIN('A') + +Note: Groups and users are granted access in the same way, but groups are prefixed with an '@' + character. In the same way, tables and namespaces are specified, but namespaces are + prefixed with an '@' character. + +For example: + + hbase> grant 'bobsmith', 'RWXCA' + hbase> grant '@admins', 'RWXCA' + hbase> grant 'bobsmith', 'RWXCA', '@ns1' + hbase> grant 'bobsmith', 'RW', 't1', 'f1', 'col1' + hbase> grant 'bobsmith', 'RW', 'ns1:t1', 'f1', 'col1' +EOF + end + + def command(*args) + + # command form is ambiguous at first argument + table_name = user = args[0] + raise(ArgumentError, "First argument should be a String") unless user.kind_of?(String) + + if args[1].kind_of?(String) + + # Original form of the command + # user in args[0] + # permissions in args[1] + # table_name in args[2] + # family in args[3] or nil + # qualifier in args[4] or nil + + permissions = args[1] + raise(ArgumentError, "Permissions are not of String type") unless permissions.kind_of?( + String) + table_name = family = qualifier = nil + table_name = args[2] # will be nil if unset + if not table_name.nil? + raise(ArgumentError, "Table name is not of String type") unless table_name.kind_of?( + String) + family = args[3] # will be nil if unset + if not family.nil? + raise(ArgumentError, "Family is not of String type") unless family.kind_of?(String) + qualifier = args[4] # will be nil if unset + if not qualifier.nil? + raise(ArgumentError, "Qualifier is not of String type") unless qualifier.kind_of?( + String) + end + end + end + format_simple_command do + security_admin.grant(user, permissions, table_name, family, qualifier) + end + + elsif args[1].kind_of?(Hash) + + # New form of the command, a cell ACL update + # table_name in args[0], a string + # a Hash mapping users (or groups) to permisisons in args[1] + # a Hash argument suitable for passing to Table#_get_scanner in args[2] + # Useful for feature testing and debugging. + + permissions = args[1] + raise(ArgumentError, "Permissions are not of Hash type") unless permissions.kind_of?(Hash) + scan = args[2] + raise(ArgumentError, "Scanner specification is not a Hash") unless scan.kind_of?(Hash) + + t = table(table_name) + now = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.list.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_permissions(put, permissions) + t.table.put(put) + end + count += 1 + end + formatter.footer(now, count) + + else + raise(ArgumentError, "Second argument should be a String or Hash") + end + + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb new file mode 100644 index 0000000000..d223a45aaf --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/incr.rb @@ -0,0 +1,65 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Incr < Command + def help + return <<-EOF +Increments a cell 'value' at specified table/row/column coordinates. +To increment a cell value in table 'ns1:t1' or 't1' at row 'r1' under column +'c1' by 1 (can be omitted) or 10 do: + + hbase> incr 'ns1:t1', 'r1', 'c1' + hbase> incr 't1', 'r1', 'c1' + hbase> incr 't1', 'r1', 'c1', 1 + hbase> incr 't1', 'r1', 'c1', 10 + hbase> incr 't1', 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> incr 't1', 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.incr 'r1', 'c1' + hbase> t.incr 'r1', 'c1', 1 + hbase> t.incr 'r1', 'c1', 10, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> t.incr 'r1', 'c1', 10, {VISIBILITY=>'PRIVATE|SECRET'} +EOF + end + + def command(table, row, column, value = nil, args = {}) + incr(table(table), row, column, value, args) + end + + def incr(table, row, column, value = nil, args={}) + format_simple_command do + if cnt = table._incr_internal(row, column, value, args) + puts "COUNTER VALUE = #{cnt}" + else + puts "No counter found at specified coordinates" + end + end + end + end + end +end + +#add incr comamnd to Table +::Hbase::Table.add_shell_command("incr") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb new file mode 100644 index 0000000000..6da7046ba7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_disabled.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsDisabled < Command + def help + return <<-EOF +Is named table disabled? For example: + hbase> is_disabled 't1' + hbase> is_disabled 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + admin.disabled?(table)? "true" : "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb new file mode 100644 index 0000000000..960ade7572 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/is_enabled.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class IsEnabled < Command + def help + return <<-EOF +Is named table enabled? For example: + hbase> is_enabled 't1' + hbase> is_enabled 'ns1:t1' +EOF + end + + def command(table) + format_simple_command do + formatter.row([ + admin.enabled?(table)? "true" : "false" + ]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb new file mode 100644 index 0000000000..dce0ae2acd --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class List < Command + def help + return <<-EOF +List all tables in hbase. Optional regular expression parameter could +be used to filter the output. Examples: + + hbase> list + hbase> list 'abc.*' + hbase> list 'ns:abc.*' + hbase> list 'ns:.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "TABLE" ]) + + list = admin.list(regex) + list.each do |table| + formatter.row([ table ]) + end + + formatter.footer(now, list.size) + return list + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb new file mode 100644 index 0000000000..6c7f99137b --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_labels.rb @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListLabels < Command + def help + return <<-EOF +List the visibility labels defined in the system. +Optional regular expression parameter could be used to filter the labels being returned. +Syntax : list_labels + +For example: + + hbase> list_labels 'secret.*' + hbase> list_labels +EOF + end + + def command(regex = ".*") + format_simple_command do + list = visibility_labels_admin.list_labels(regex) + list.each do |label| + formatter.row([org.apache.hadoop.hbase.util.Bytes::toStringBinary(label.toByteArray)]) + end + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb new file mode 100644 index 0000000000..5d25604afa --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace.rb @@ -0,0 +1,46 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListNamespace < Command + def help + return <<-EOF +List all namespaces in hbase. Optional regular expression parameter could +be used to filter the output. Examples: + + hbase> list_namespace + hbase> list_namespace 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "NAMESPACE" ]) + + list = admin.list_namespace(regex) + list.each do |table| + formatter.row([ table ]) + end + + formatter.footer(now, list.size) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb new file mode 100644 index 0000000000..29e18123e6 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_namespace_tables.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListNamespaceTables < Command + def help + return <<-EOF +List all tables that are members of the namespace. +Examples: + + hbase> list_namespace_tables 'ns1' +EOF + end + + def command(namespace) + now = Time.now + formatter.header([ "TABLE" ]) + + list = admin.list_namespace_tables(namespace) + list.each do |table| + formatter.row([ table ]) + end + + formatter.footer(now, list.size) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb new file mode 100644 index 0000000000..cc1be04416 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_peers.rb @@ -0,0 +1,48 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListPeers< Command + def help + return <<-EOF +List all replication peer clusters. + + hbase> list_peers +EOF + end + + def command() + now = Time.now + peers = replication_admin.list_peers + + formatter.header(["PEER_ID", "CLUSTER_KEY", "STATE", "TABLE_CFS"]) + + peers.entrySet().each do |e| + state = replication_admin.get_peer_state(e.key) + tableCFs = replication_admin.show_peer_tableCFs(e.key) + formatter.row([ e.key, e.value, state, tableCFs ]) + end + + formatter.footer(now) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb new file mode 100644 index 0000000000..0db1d83aeb --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_replicated_tables.rb @@ -0,0 +1,50 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListReplicatedTables< Command + def help + return <<-EOF +List all the tables and column families replicated from this cluster + + hbase> list_replicated_tables + hbase> list_replicated_tables 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + + formatter.header([ "TABLE:COLUMNFAMILY", "ReplicationType" ], [ 32 ]) + list = replication_admin.list_replicated_tables(regex) + list.each do |e| + if e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONTYPE) == org.apache.hadoop.hbase.client.replication.ReplicationAdmin::REPLICATIONGLOBAL + replicateType = "GLOBAL" + else + replicateType = "unknown" + end + formatter.row([e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::TNAME) + ":" + e.get(org.apache.hadoop.hbase.client.replication.ReplicationAdmin::CFNAME), replicateType], true, [32]) + end + formatter.footer(now) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb new file mode 100644 index 0000000000..4e68802f8d --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/list_snapshots.rb @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'time' + +module Shell + module Commands + class ListSnapshots < Command + def help + return <<-EOF +List all snapshots taken (by printing the names and relative information). +Optional regular expression parameter could be used to filter the output +by snapshot name. + +Examples: + hbase> list_snapshots + hbase> list_snapshots 'abc.*' +EOF + end + + def command(regex = ".*") + now = Time.now + formatter.header([ "SNAPSHOT", "TABLE + CREATION TIME"]) + + list = admin.list_snapshot(regex) + list.each do |snapshot| + creation_time = Time.at(snapshot.getCreationTime() / 1000).to_s + formatter.row([ snapshot.getName, snapshot.getTable + " (" + creation_time + ")" ]) + end + + formatter.footer(now, list.size) + return list.map { |s| s.getName() } + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb new file mode 100644 index 0000000000..825748336c --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/major_compact.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MajorCompact < Command + def help + return <<-EOF + Run major compaction on passed table or pass a region row + to major compact an individual region. To compact a single + column family within a region specify the region name + followed by the column family name. + Examples: + Compact all regions in a table: + hbase> major_compact 't1' + hbase> major_compact 'ns1:t1' + Compact an entire region: + hbase> major_compact 'r1' + Compact a single column family within a region: + hbase> major_compact 'r1', 'c1' + Compact a single column family within a table: + hbase> major_compact 't1', 'c1' + EOF + end + + def command(table_or_region_name, family = nil) + format_simple_command do + admin.major_compact(table_or_region_name, family) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb new file mode 100644 index 0000000000..6afa2e5b50 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/merge_region.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class MergeRegion < Command + def help + return <<-EOF +Merge two regions. Passing 'true' as the optional third parameter will force +a merge ('force' merges regardless else merge will fail unless passed +adjacent regions. 'force' is for expert use only). + +NOTE: You must pass the encoded region name, not the full region name so +this command is a little different from other region operations. The encoded +region name is the hash suffix on region names: e.g. if the region name were +TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then +the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 + +Examples: + + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME' + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', true +EOF + end + + def command(encoded_region_a_name, encoded_region_b_name, force = 'false') + format_simple_command do + admin.merge_region(encoded_region_a_name, encoded_region_b_name, force) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb new file mode 100644 index 0000000000..e6b28288e3 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/move.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Move < Command + def help + return <<-EOF +Move a region. Optionally specify target regionserver else we choose one +at random. NOTE: You pass the encoded region name, not the region name so +this command is a little different to the others. The encoded region name +is the hash suffix on region names: e.g. if the region name were +TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then +the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 +A server name is its host, port plus startcode. For example: +host187.example.com,60020,1289493121758 +Examples: + + hbase> move 'ENCODED_REGIONNAME' + hbase> move 'ENCODED_REGIONNAME', 'SERVER_NAME' +EOF + end + + def command(encoded_region_name, server_name = nil) + format_simple_command do + admin.move(encoded_region_name, server_name) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb new file mode 100644 index 0000000000..2b47a4d5c6 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/put.rb @@ -0,0 +1,57 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Put < Command + def help + return <<-EOF +Put a cell 'value' at specified table/row/column and optionally +timestamp coordinates. To put a cell value into table 'ns1:t1' or 't1' +at row 'r1' under column 'c1' marked with the time 'ts1', do: + + hbase> put 'ns1:t1', 'r1', 'c1', 'value' + hbase> put 't1', 'r1', 'c1', 'value' + hbase> put 't1', 'r1', 'c1', 'value', ts1 + hbase> put 't1', 'r1', 'c1', 'value', {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} + hbase> put 't1', 'r1', 'c1', 'value', ts1, {VISIBILITY=>'PRIVATE|SECRET'} + +The same commands also can be run on a table reference. Suppose you had a reference +t to table 't1', the corresponding command would be: + + hbase> t.put 'r1', 'c1', 'value', ts1, {ATTRIBUTES=>{'mykey'=>'myvalue'}} +EOF + end + + def command(table, row, column, value, timestamp=nil, args = {}) + put table(table), row, column, value, timestamp, args + end + + def put(table, row, column, value, timestamp = nil, args = {}) + format_simple_command do + table._put_internal(row, column, value, timestamp, args) + end + end + end + end +end + +#Add the method table.put that calls Put.put +::Hbase::Table.add_shell_command("put") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb new file mode 100644 index 0000000000..5ae5786fc4 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeer< Command + def help + return <<-EOF +Stops the specified replication stream and deletes all the meta +information kept about it. Examples: + + hbase> remove_peer '1' +EOF + end + + def command(id) + format_simple_command do + replication_admin.remove_peer(id) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb new file mode 100644 index 0000000000..5b15b52965 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/remove_peer_tableCFs.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RemovePeerTableCFs < Command + def help + return <<-EOF +Remove a table / table-cf from the table-cfs config for the specified peer +Examples: + + # Remove a table / table-cf from the replicable table-cfs for a peer + hbase> remove_peer_tableCFs '2', "table1" + hbase> remove_peer_tableCFs '2', "table1:cf1" + +EOF + end + + def command(id, table_cfs) + format_simple_command do + replication_admin.remove_peer_tableCFs(id, table_cfs) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb new file mode 100644 index 0000000000..4d531711bc --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/restore_snapshot.rb @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RestoreSnapshot < Command + def help + return <<-EOF +Restore a specified snapshot. +The restore will replace the content of the original table, +bringing back the content to the snapshot state. +The table must be disabled. + +Examples: + hbase> restore_snapshot 'snapshotName' +EOF + end + + def command(snapshot_name) + format_simple_command do + admin.restore_snapshot(snapshot_name) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb new file mode 100644 index 0000000000..768989b6c0 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/revoke.rb @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Revoke < Command + def help + return <<-EOF +Revoke a user's access rights. +Syntax : revoke [<@namespace> [
[ []]]] + +Note: Groups and users access are revoked in the same way, but groups are prefixed with an '@' + character. In the same way, tables and namespaces are specified, but namespaces are + prefixed with an '@' character. + +For example: + + hbase> revoke 'bobsmith' + hbase> revoke '@admins' + hbase> revoke 'bobsmith', '@ns1' + hbase> revoke 'bobsmith', 't1', 'f1', 'col1' + hbase> revoke 'bobsmith', 'ns1:t1', 'f1', 'col1' +EOF + end + + def command(user, table_name=nil, family=nil, qualifier=nil) + format_simple_command do + security_admin.revoke(user, table_name, family, qualifier) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb new file mode 100644 index 0000000000..7e28b20b12 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/scan.rb @@ -0,0 +1,115 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Scan < Command + def help + return <<-EOF +Scan a table; pass table name and optionally a dictionary of scanner +specifications. Scanner specifications may include one or more of: +TIMERANGE, FILTER, LIMIT, STARTROW, STOPROW, TIMESTAMP, MAXLENGTH, +or COLUMNS, CACHE or RAW, VERSIONS + +If no columns are specified, all columns will be scanned. +To scan all members of a column family, leave the qualifier empty as in +'col_family:'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +Some examples: + + hbase> scan 'hbase:meta' + hbase> scan 'hbase:meta', {COLUMNS => 'info:regioninfo'} + hbase> scan 'ns1:t1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], LIMIT => 10, STARTROW => 'xyz'} + hbase> scan 't1', {COLUMNS => 'c1', TIMERANGE => [1303668804, 1303668904]} + hbase> scan 't1', {REVERSED => true} + hbase> scan 't1', {FILTER => "(PrefixFilter ('row2') AND + (QualifierFilter (>=, 'binary:xyz'))) AND (TimestampsFilter ( 123, 456))"} + hbase> scan 't1', {FILTER => + org.apache.hadoop.hbase.filter.ColumnPaginationFilter.new(1, 0)} + hbase> scan 't1', {CONSISTENCY => 'TIMELINE'} +For setting the Operation Attributes + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], ATTRIBUTES => {'mykey' => 'myvalue'}} + hbase> scan 't1', { COLUMNS => ['c1', 'c2'], AUTHORIZATIONS => ['PRIVATE','SECRET']} +For experts, there is an additional option -- CACHE_BLOCKS -- which +switches block caching for the scanner on (true) or off (false). By +default it is enabled. Examples: + + hbase> scan 't1', {COLUMNS => ['c1', 'c2'], CACHE_BLOCKS => false} + +Also for experts, there is an advanced option -- RAW -- which instructs the +scanner to return all cells (including delete markers and uncollected deleted +cells). This option cannot be combined with requesting specific COLUMNS. +Disabled by default. Example: + + hbase> scan 't1', {RAW => true, VERSIONS => 10} + +Besides the default 'toStringBinary' format, 'scan' supports custom formatting +by column. A user can define a FORMATTER by adding it to the column name in +the scan specification. The FORMATTER can be stipulated: + + 1. either as a org.apache.hadoop.hbase.util.Bytes method name (e.g, toInt, toString) + 2. or as a custom class followed by method name: e.g. 'c(MyFormatterClass).format'. + +Example formatting cf:qualifier1 and cf:qualifier2 both as Integers: + hbase> scan 't1', {COLUMNS => ['cf:qualifier1:toInt', + 'cf:qualifier2:c(org.apache.hadoop.hbase.util.Bytes).toInt'] } + +Note that you can specify a FORMATTER by column only (cf:qualifier). You cannot +specify a FORMATTER for all columns of a column family. + +Scan can also be used directly from a table, by first getting a reference to a +table, like such: + + hbase> t = get_table 't' + hbase> t.scan + +Note in the above situation, you can still provide all the filtering, columns, +options, etc as described above. + +EOF + end + + def command(table, args = {}) + scan(table(table), args) + end + + #internal command that actually does the scanning + def scan(table, args = {}) + now = Time.now + formatter.header(["ROW", "COLUMN+CELL"]) + + #actually do the scanning + count = table._scan_internal(args) do |row, cells| + formatter.row([ row, cells ]) + end + + formatter.footer(now, count) + end + end + end +end + +#Add the method table.scan that calls Scan.scan +::Hbase::Table.add_shell_command("scan") diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb new file mode 100644 index 0000000000..4a52eb0ef7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_auths.rb @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetAuths < Command + def help + return <<-EOF +Add a set of visibility labels for a user or group +Syntax : set_auths 'user',[label1, label2] + +For example: + + hbase> set_auths 'user1', ['SECRET','PRIVATE'] + hbase> set_auths '@group1', ['SECRET','PRIVATE'] +EOF + end + + def command(user, *args) + format_simple_command do + visibility_labels_admin.set_auths(user, args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb new file mode 100644 index 0000000000..3a88dbb741 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_peer_tableCFs.rb @@ -0,0 +1,47 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetPeerTableCFs< Command + def help + return <<-EOF + Set the replicable table-cf config for the specified peer + Examples: + + # set all tables to be replicable for a peer + hbase> set_peer_tableCFs '1', "" + hbase> set_peer_tableCFs '1' + # set table / table-cf to be replicable for a peer, for a table without + # an explicit column-family list, all replicable column-families (with + # replication_scope == 1) will be replicated + hbase> set_peer_tableCFs '2', "table1; table2:cf1,cf2; table3:cfA,cfB" + + EOF + end + + def command(id, peer_table_cfs = nil) + format_simple_command do + replication_admin.set_peer_tableCFs(id, peer_table_cfs) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb new file mode 100644 index 0000000000..b2b57b19c3 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/set_visibility.rb @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetVisibility < Command + def help + return <<-EOF +Set the visibility expression on one or more existing cells. + +Pass table name, visibility expression, and a dictionary containing +scanner specifications. Scanner specifications may include one or more +of: TIMERANGE, FILTER, STARTROW, STOPROW, TIMESTAMP, or COLUMNS + +If no columns are specified, all columns will be included. +To include all members of a column family, leave the qualifier empty as in +'col_family:'. + +The filter can be specified in two ways: +1. Using a filterString - more information on this is available in the +Filter Language document attached to the HBASE-4176 JIRA +2. Using the entire package name of the filter. + +Examples: + + hbase> set_visibility 't1', 'A|B', {COLUMNS => ['c1', 'c2']} + hbase> set_visibility 't1', '(A&B)|C', {COLUMNS => 'c1', + TIMERANGE => [1303668804, 1303668904]} + hbase> set_visibility 't1', 'A&B&C', {FILTER => "(PrefixFilter ('row2') AND + (QualifierFilter (>=, 'binary:xyz'))) AND + (TimestampsFilter ( 123, 456))"} + +This command will only affect existing cells and is expected to be mainly +useful for feature testing and functional verification. +EOF + end + + def command(table, visibility, scan) + t = table(table) + now = Time.now + scanner = t._get_scanner(scan) + count = 0 + iter = scanner.iterator + while iter.hasNext + row = iter.next + row.list.each do |cell| + put = org.apache.hadoop.hbase.client.Put.new(row.getRow) + put.add(cell) + t.set_cell_visibility(put, visibility) + t.table.put(put) + end + count += 1 + end + formatter.footer(now, count) + end + + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb new file mode 100644 index 0000000000..cdbd9ed9b5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_filters.rb @@ -0,0 +1,49 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +java_import org.apache.hadoop.hbase.filter.ParseFilter + +module Shell + module Commands + class ShowFilters < Command + def help + return <<-EOF +Show all the filters in hbase. Example: + hbase> show_filters + + ColumnPrefixFilter + TimestampsFilter + PageFilter + ..... + KeyOnlyFilter +EOF + end + + def command( ) + now = Time.now + parseFilter = ParseFilter.new + supportedFilters = parseFilter.getSupportedFilters + + supportedFilters.each do |filter| + formatter.row([filter]) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb new file mode 100644 index 0000000000..3ce3d06f01 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/show_peer_tableCFs.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ShowPeerTableCFs< Command + def help + return <<-EOF + Show replicable table-cf config for the specified peer. + + hbase> show_peer_tableCFs + EOF + end + + def command(id) + puts replication_admin.show_peer_tableCFs(id) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb new file mode 100644 index 0000000000..15bf298e04 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/snapshot.rb @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Snapshot < Command + def help + return <<-EOF +Take a snapshot of specified table. Examples: + + hbase> snapshot 'sourceTable', 'snapshotName' + hbase> snapshot 'namespace:sourceTable', 'snapshotName', {SKIP_FLUSH => true} +EOF + end + + def command(table, snapshot_name, *args) + format_simple_command do + admin.snapshot(table, snapshot_name, *args) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb new file mode 100644 index 0000000000..9dc424ff68 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/split.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Split < Command + def help + return <<-EOF +Split entire table or pass a region to split individual region. With the +second parameter, you can specify an explicit split key for the region. +Examples: + split 'tableName' + split 'namespace:tableName' + split 'regionName' # format: 'tableName,startKey,id' + split 'tableName', 'splitKey' + split 'regionName', 'splitKey' +EOF + end + + def command(table_or_region_name, split_point = nil) + format_simple_command do + admin.split(table_or_region_name, split_point) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb new file mode 100644 index 0000000000..b22b272398 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/status.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Status < Command + def help + return <<-EOF +Show cluster status. Can be 'summary', 'simple', 'detailed', or 'replication'. The +default is 'summary'. Examples: + + hbase> status + hbase> status 'simple' + hbase> status 'summary' + hbase> status 'detailed' + hbase> status 'replication' + hbase> status 'replication', 'source' + hbase> status 'replication', 'sink' +EOF + end + + def command(format = 'summary',type = 'both') + admin.status(format, type) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb new file mode 100644 index 0000000000..e5a1858a92 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/table_help.rb @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TableHelp < Command + def help + return Hbase::Table.help + end + + #just print the help + def command + # call the shell to get the nice formatting there + @shell.help_command 'table_help' + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb new file mode 100644 index 0000000000..5e0093092f --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/trace.rb @@ -0,0 +1,76 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +HTrace = org.apache.htrace.Trace +java_import org.apache.htrace.Sampler +java_import org.apache.hadoop.hbase.trace.SpanReceiverHost + +module Shell + module Commands + class Trace < Command + def help + return <<-EOF +Start or Stop tracing using HTrace. +Always returns true if tracing is running, otherwise false. +If the first argument is 'start', new span is started. +If the first argument is 'stop', current running span is stopped. +('stop' returns false on success.) +If the first argument is 'status', just returns if or not tracing is running. +On 'start'-ing, you can optionally pass the name of span as the second argument. +The default name of span is 'HBaseShell'. +Repeating 'start' does not start nested span. + +Examples: + + hbase> trace 'start' + hbase> trace 'status' + hbase> trace 'stop' + + hbase> trace 'start', 'MySpanName' + hbase> trace 'stop' + +EOF + end + + def command(startstop="status", spanname="HBaseShell") + format_and_return_simple_command do + trace(startstop, spanname) + end + end + + def trace(startstop, spanname) + @@receiver ||= SpanReceiverHost.getInstance(@shell.hbase.configuration) + if startstop == "start" + if not tracing? + @@tracescope = HTrace.startSpan(spanname, Sampler.ALWAYS) + end + elsif startstop == "stop" + if tracing? + @@tracescope.close() + end + end + tracing? + end + + def tracing?() + HTrace.isTracing() + end + + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb new file mode 100644 index 0000000000..b7812fbe3d --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Truncate < Command + def help + return <<-EOF + Disables, drops and recreates the specified table. +EOF + end + + def command(table) + format_simple_command do + puts "Truncating '#{table}' table (it may take a while):" + admin.truncate(table) { |log| puts " - #{log}" } + end + end + + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb new file mode 100644 index 0000000000..918b23289a --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/truncate_preserve.rb @@ -0,0 +1,38 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class TruncatePreserve < Command + def help + return <<-EOF + Disables, drops and recreates the specified table while still maintaing the previous region boundaries. +EOF + end + + def command(table) + format_simple_command do + puts "Truncating '#{table}' table (it may take a while):" + admin.truncate_preserve(table) { |log| puts " - #{log}" } + end + end + + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb new file mode 100644 index 0000000000..5eea71f6e0 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/unassign.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Unassign < Command + def help + return <<-EOF +Unassign a region. Unassign will close region in current location and then +reopen it again. Pass 'true' to force the unassignment ('force' will clear +all in-memory state in master before the reassign. If results in +double assignment use hbck -fix to resolve. To be used by experts). +Use with caution. For expert use only. Examples: + + hbase> unassign 'REGIONNAME' + hbase> unassign 'REGIONNAME', true + hbase> unassign 'ENCODED_REGIONNAME' + hbase> unassign 'ENCODED_REGIONNAME', true +EOF + end + + def command(region_name, force = 'false') + format_simple_command do + admin.unassign(region_name, force) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb new file mode 100644 index 0000000000..05295b7996 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_all_config.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateAllConfig < Command + def help + return <<-EOF +Reload a subset of configuration on all servers in the cluster. See +http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_all_config +EOF + end + + def command() + format_simple_command do + admin.update_all_config() + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb new file mode 100644 index 0000000000..9f65fddd25 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/update_config.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateConfig < Command + def help + return <<-EOF +Reload a subset of configuration on server 'servername' where servername is +host, port plus startcode. For example: host187.example.com,60020,1289493121758 +See http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_config 'servername' +EOF + end + + def command(serverName) + format_simple_command do + admin.update_config(serverName) + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb new file mode 100644 index 0000000000..e4673fc391 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/user_permission.rb @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UserPermission < Command + def help + return <<-EOF +Show all permissions for the particular user. +Syntax : user_permission
+ +Note: A namespace must always precede with '@' character. + +For example: + + hbase> user_permission + hbase> user_permission '@ns1' + hbase> user_permission 'table1' + hbase> user_permission 'namespace1:table1' + hbase> user_permission '.*' + hbase> user_permission '^[A-C].*' +EOF + end + + def command(table_regex=nil) + #format_simple_command do + #admin.user_permission(table_regex) + now = Time.now + formatter.header(["User", "Namespace,Table,Family,Qualifier:Permission"]) + + count = security_admin.user_permission(table_regex) do |user, permission| + formatter.row([ user, permission]) + end + + formatter.footer(now, count) + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb new file mode 100644 index 0000000000..63e9712a52 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/version.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Version < Command + def help + return <<-EOF +Output this HBase version +EOF + end + + def command + # Output version. + puts "#{org.apache.hadoop.hbase.util.VersionInfo.getVersion()}, " + + "r#{org.apache.hadoop.hbase.util.VersionInfo.getRevision()}, " + + "#{org.apache.hadoop.hbase.util.VersionInfo.getDate()}" + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb new file mode 100644 index 0000000000..0fe1870af2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/wal_roll.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module Shell + module Commands + class WalRoll < Command + def help + return <<-EOF +Roll the log writer. That is, start writing log messages to a new file. +The name of the regionserver should be given as the parameter. A +'server_name' is the host, port plus startcode of a regionserver. For +example: host187.example.com,60020,1289493121758 (find servername in +master ui or when you do detailed status in shell) +EOF + end + + def command(server_name) + format_simple_command do + admin.wal_roll(server_name) + end + end + end + #TODO remove old HLog version + class HlogRoll < WalRoll + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb new file mode 100644 index 0000000000..3b6b32d257 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/whoami.rb @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Whoami < Command + def help + return <<-EOF +Show the current hbase user. +Syntax : whoami +For example: + + hbase> whoami +EOF + end + + def command() + user = org.apache.hadoop.hbase.security.User.getCurrent() + puts "#{user.toString()}" + groups = user.getGroupNames().to_a + if not groups.nil? and groups.length > 0 + puts " groups: #{groups.join(', ')}" + end + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb new file mode 100644 index 0000000000..c0b509a3c6 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/commands/zk_dump.rb @@ -0,0 +1,34 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ZkDump < Command + def help + return <<-EOF +Dump status of HBase cluster as seen by ZooKeeper. +EOF + end + + def command + puts admin.zk_dump + end + end + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb new file mode 100644 index 0000000000..36aaf76aea --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/ruby/shell/formatter.rb @@ -0,0 +1,170 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Results formatter +module Shell + module Formatter + # Base abstract class for results formatting. + class Base + attr_reader :row_count + + def is_valid_io?(obj) + obj.instance_of?(IO) || obj == Kernel + end + + def refresh_width() + if $stdout.tty? + @max_width = Java::jline.Terminal.getTerminal().getTerminalWidth() + else + @max_width = 0 + end + end + + # Takes an output stream and a print width. + def initialize(opts = {}) + options = { + :output_stream => Kernel, + }.merge(opts) + + @out = options[:output_stream] + refresh_width + @row_count = 0 + + # raise an error if the stream is not valid + raise(TypeError, "Type #{@out.class} of parameter #{@out} is not IO") unless is_valid_io?(@out) + end + + def header(args = [], widths = []) + refresh_width + row(args, false, widths) if args.length > 0 + @row_count = 0 + end + + # Output a row. + # Inset is whether or not to offset row by a space. + def row(args = [], inset = true, widths = []) + # Print out nothing + return if !args || args.empty? + + # Print a string + if args.is_a?(String) + output(args) + @out.puts + return + end + + # TODO: Look at the type. Is it RowResult? + if args.length == 1 + splits = split(@max_width, dump(args[0])) + for l in splits + output(@max_width, l) + @out.puts + end + elsif args.length == 2 + if @max_width == 0 + col1width = col2width = 0 + else + col1width = (not widths or widths.length == 0) ? @max_width / 4 : @max_width * widths[0] / 100 + col2width = (not widths or widths.length < 2) ? @max_width - col1width - 2 : @max_width * widths[1] / 100 - 2 + end + splits1 = split(col1width, dump(args[0])) + splits2 = split(col2width, dump(args[1])) + biggest = (splits2.length > splits1.length)? splits2.length: splits1.length + index = 0 + while index < biggest + # Inset by one space if inset is set. + @out.print(" ") if inset + output(col1width, splits1[index]) + # Add extra space so second column lines up w/ second column output + @out.print(" ") unless inset + @out.print(" ") + output(col2width, splits2[index]) + index += 1 + @out.puts + end + else + # Print a space to set off multi-column rows + print ' ' + first = true + for e in args + @out.print " " unless first + first = false + @out.print e + end + puts + end + @row_count += 1 + end + + def split(width, str) + if width == 0 + return [str] + end + result = [] + index = 0 + while index < str.length do + result << str.slice(index, width) + index += width + end + result + end + + def dump(str) + return if str.instance_of?(Fixnum) + # Remove double-quotes added by 'dump'. + return str + end + + def output(str) + output(@max_width, str) + end + + def output(width, str) + if str == nil + str = '' + end + if not width or width == str.length + @out.print(str) + else + @out.printf('%-*s', width, str) + end + end + + def footer(start_time = nil, row_count = nil) + return unless start_time + row_count ||= @row_count + # Only output elapsed time and row count if startTime passed + @out.puts("%d row(s) in %.4f seconds" % [row_count, Time.now - start_time]) + end + end + + + class Console < Base + end + + class XHTMLFormatter < Base + # http://www.germane-software.com/software/rexml/doc/classes/REXML/Document.html + # http://www.crummy.com/writing/RubyCookbook/test_results/75942.html + end + + class JSON < Base + end + end +end + diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb new file mode 100755 index 0000000000..9287e5bfc2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/shutdown_regionserver.rb @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script is used to issue a stop command to a regionserver via RPC. +# Intended for use in environments where sshing around is inappropriate +# Run it like this by passing it to a jruby interpreter: +# +# ./bin/hbase org.jruby.Main bin/shutdown_regionserver.rb c2021:16020 + +include Java +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.client.HBaseAdmin + +def usage(msg=nil) + $stderr.puts 'Usage: shutdown_regionserver.rb ..' + $stderr.puts + $stderr.puts 'Stops the specified regionservers via RPC' + $stderr.puts 'Error: %s' % msg if msg + abort +end + +usage if ARGV.length < 1 + +ARGV.each do |x| + usage 'Invalid host:port: %s' % x unless x.include? ':' +end + +config = HBaseConfiguration.create() +begin + admin = HBaseAdmin.new(config) +rescue + abort "Error: Couldn't instantiate HBaseAdmin" +end + +ARGV.each do |hostport| + admin.stopRegionServer(hostport) +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd new file mode 100755 index 0000000000..676a11e6e2 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.cmd @@ -0,0 +1,61 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Modelled after $HADOOP_HOME/bin/start-hbase.sh. + +@rem Start hadoop hbase daemons. +@rem Run this on master node. +@echo off +set usage="Usage: start-hbase.cmd" + +setlocal + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) + +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call %hbase-config-script% + +set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed +for /f %%i in ('%distModeCommand%') do set distMode=%%i + +if "%distMode%"=="false" ( + start "HBase Distribution" %HBASE_BIN_PATH%\hbase.cmd master start +) else ( + if "%distMode%"=="true" ( + @echo This is not implemented yet. Stay tuned. + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "${HBASE_CONF_DIR}" start zookeeper + @rem call %HBASE_BIN_PATH%\hbase-daemon.cmd --config "${HBASE_CONF_DIR}" start master + + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_REGIONSERVERS%" start regionserver + @rem call %HBASE_BIN_PATH%\hbase-daemons.cmd --config "%HBASE_CONF_DIR%" --hosts "%HBASE_BACKUP_MASTERS%" start master-backup + ) else ( + echo ERROR: Could not determine the startup mode. + ) +) + +@rem -------------- End of main script -------------- +endlocal +goto :eof \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh new file mode 100755 index 0000000000..aeb9469fb7 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/start-hbase.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/start-hbase.sh. + +# Start hadoop hbase daemons. +# Run this on master node. +usage="Usage: start-hbase.sh" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi + + +if [ "$1" = "autorestart" ] +then + commandToRun="autorestart" +else + commandToRun="start" +fi + +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` + + +if [ "$distMode" == 'false' ] +then + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master $@ +else + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" $commandToRun zookeeper + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" $commandToRun master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" $commandToRun regionserver + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" $commandToRun master-backup +fi diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd new file mode 100755 index 0000000000..97180554b5 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.cmd @@ -0,0 +1,54 @@ +@echo off +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Modelled after $HADOOP_HOME/bin/stop-hbase.sh. + +@rem Stop hadoop hbase daemons. Run this on master node. + +setlocal + +for %%i in (%0) do ( + if not defined HBASE_BIN_PATH ( + set HBASE_BIN_PATH=%%~dpi + ) +) + +if "%HBASE_BIN_PATH:~-1%" == "\" ( + set HBASE_BIN_PATH=%HBASE_BIN_PATH:~0,-1% +) +set hbase-config-script=%HBASE_BIN_PATH%\hbase-config.cmd +call %hbase-config-script% + +set distModeCommand=call %HBASE_BIN_PATH%\hbase.cmd org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed +for /f %%i in ('%distModeCommand%') do set distMode=%%i + +if "%distMode%"=="false" ( + call %HBASE_BIN_PATH%\hbase.cmd master stop + +) else ( + if "%distMode%"=="true" ( + @echo This is not implemented yet. Stay tuned. + ) else ( + echo ERROR: Could not determine the startup mode. + ) +) + +@rem -------------- End of main script -------------- +endlocal +goto :eof \ No newline at end of file diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh new file mode 100755 index 0000000000..8b81deedd8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/stop-hbase.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. + +# Stop hadoop hbase daemons. Run this on master node. + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh +. "$bin"/hbase-common.sh + +# variables needed for stop command +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid + +echo -n stopping hbase +echo "`date` Stopping hbase (via master)" >> $loglog + +nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + master stop "$@" > "$logout" 2>&1 < /dev/null & + +waitForProcessEnd `cat $pid` 'stop-master-command' + +rm -f $pid + + +# distributed == false means that the HMaster will kill ZK when it exits +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'true' ] +then + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper +fi diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh new file mode 100755 index 0000000000..eb8633f502 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/test/process_based_cluster.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# +#/** +# * Copyright The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e -u -o pipefail + +SCRIPT_NAME=${0##*/} +SCRIPT_DIR=$(cd `dirname $0` && pwd ) + +print_usage() { + cat >&2 < +Options: + --kill + Kill local process-based HBase cluster using pid files. + --show + Show HBase processes running on this machine +EOT + exit 1 +} + +show_processes() { + ps -ef | grep -P "(HRegionServer|HMaster|HQuorumPeer) start" | grep -v grep +} + +cmd_specified() { + if [ "$CMD_SPECIFIED" ]; then + echo "Only one command can be specified" >&2 + exit 1 + fi + CMD_SPECIFIED=1 +} + +list_pid_files() { + LOCAL_CLUSTER_DIR=$SCRIPT_DIR/../../target/local_cluster + LOCAL_CLUSTER_DIR=$( cd $LOCAL_CLUSTER_DIR && pwd ) + find $LOCAL_CLUSTER_DIR -name "*.pid" +} + +if [ $# -eq 0 ]; then + print_usage +fi + +IS_KILL="" +IS_SHOW="" +CMD_SPECIFIED="" + +while [ $# -ne 0 ]; do + case "$1" in + -h|--help) + print_usage ;; + --kill) + IS_KILL=1 + cmd_specified ;; + --show) + IS_SHOW=1 + cmd_specified ;; + *) + echo "Invalid option: $1" >&2 + exit 1 + esac + shift +done + +if [ "$IS_KILL" ]; then + list_pid_files | \ + while read F; do + PID=`cat $F` + echo "Killing pid $PID from file $F" + # Kill may fail but that's OK, so turn off error handling for a moment. + set +e + kill -9 $PID + set -e + done +elif [ "$IS_SHOW" ]; then + PIDS="" + for F in `list_pid_files`; do + PID=`cat $F` + if [ -n "$PID" ]; then + if [ -n "$PIDS" ]; then + PIDS="$PIDS," + fi + PIDS="$PIDS$PID" + fi + done + ps -p $PIDS +else + echo "No command specified" >&2 + exit 1 +fi + + diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb b/titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb new file mode 100755 index 0000000000..68140124c8 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/thread-pool.rb @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. + +require 'thread' + +class ThreadPool + def initialize(poolsize) + @queue = Queue.new + @poolsize = poolsize + @pool = Array.new(@poolsize) do |i| + Thread.new do + Thread.current[:id] = i + catch(:close) do + loop do + job, args = @queue.pop + job.call(*args) + end + end + end + end + end + + def launch(*args, &block) + @queue << [block, args] + end + + def stop + @poolsize.times do + launch { throw :close } + end + @pool.map(&:join) + end +end diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch b/titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch new file mode 100644 index 0000000000..afdd6d097c --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/titan_customizations_to_hbase_script.patch @@ -0,0 +1,50 @@ +--- /home/dalaro/hbase-1.0.2/bin/hbase 2015-08-25 18:34:24.000000000 -0400 ++++ hbase 2015-09-02 15:37:46.000000000 -0400 +@@ -168,6 +168,31 @@ + } + + ++add_maven_deps_to_classpath() { ++ f="${HBASE_HOME}/target/cached_classpath.txt" ++ if [ ! -f "${f}" ]; then ++ # ------------------- ++ # Titan build support ++ # ------------------- ++ # Check for Maven command line options file. ++ # If titan-hbase has already been built, then this file will contain options ++ # to target the version of Hadoop used during compilation. This is ++ # necessary when the user compiles titan-hbase with a non-default Hadoop ++ # version (e.g. 2.2.0). If we omitted this check, then we'd try to start ++ # HBase with the default Hadoop version (currentl 1.0.4), and it would break. ++ MVN_OPTS= ++ optf="${HBASE_HOME}/target/test-classes/mvn_opts.txt" ++ unset IFS ++ if [ -f "${optf}" ]; then ++ MVN_OPTS=`cat "${optf}"` ++ fi ++ mvn -f "${HBASE_HOME}/pom.xml" $MVN_OPTS dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null ++ IFS= ++ fi ++ CLASSPATH=${CLASSPATH}:`cat "${f}"` ++} ++ ++ + #Add the development env class path stuff + if $in_dev_env; then + add_maven_deps_to_classpath +@@ -286,13 +311,8 @@ + CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" + HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" + fi +- #find the hbase ruby sources +- if [ -d "$HBASE_HOME/lib/ruby" ]; then +- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/lib/ruby" +- else +- HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/hbase-shell/src/main/ruby" +- fi +- HBASE_OPTS="$HBASE_OPTS $HBASE_SHELL_OPTS" ++ # find the hbase ruby sources ++ HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/bin/ruby" + CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" + elif [ "$COMMAND" = "hbck" ] ; then + CLASS='org.apache.hadoop.hbase.util.HBaseFsck' diff --git a/titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh b/titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh new file mode 100755 index 0000000000..97bf41b605 --- /dev/null +++ b/titan-hbase-parent/titan-hbase-11/src/test/bin/zookeepers.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all zookeeper hosts. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: zookeepers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +if [ "$HBASE_MANAGES_ZK" = "" ]; then + HBASE_MANAGES_ZK=true +fi + +if [ "$HBASE_MANAGES_ZK" = "true" ]; then + hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` + cmd=$"${@// /\\ }" + for zookeeper in $hosts; do + ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait