From 22dd68c855efc3c718a1b17715937ccb0a611b9c Mon Sep 17 00:00:00 2001
From: Honestman <honestmanxin@gmail.com>
Date: Fri, 22 Jan 2016 22:00:07 +0800
Subject: [PATCH] Text.tokenize add unicode support ,fix Bug when str.length()
 == 1 the return tokens is empty list

---
 .../java/com/thinkaurelius/titan/core/attribute/Text.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Text.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Text.java
index 4b247d84e6..04779151ef 100644
--- a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Text.java
+++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Text.java
@@ -166,12 +166,12 @@ public static List<String> tokenize(String str) {
         ArrayList<String> tokens = new ArrayList<String>();
         int previous = 0;
         for (int p = 0; p < str.length(); p++) {
-            if (!Character.isLetterOrDigit(str.charAt(p))) {
+            if (!Character.isLetterOrDigit(str.codePointAt(p))) {
                 if (p > previous + MIN_TOKEN_LENGTH) tokens.add(str.substring(previous, p));
                 previous = p + 1;
             }
         }
-        if (previous + MIN_TOKEN_LENGTH < str.length()) tokens.add(str.substring(previous, str.length()));
+        if (previous + MIN_TOKEN_LENGTH <= str.length()) tokens.add(str.substring(previous, str.length()));
         return tokens;
     }