1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/rt/emul/compact/src/main/java/java/util/StringTokenizer.java Tue Feb 26 16:54:16 2013 +0100
1.3 @@ -0,0 +1,431 @@
1.4 +/*
1.5 + * Copyright (c) 1994, 2004, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation. Oracle designates this
1.11 + * particular file as subject to the "Classpath" exception as provided
1.12 + * by Oracle in the LICENSE file that accompanied this code.
1.13 + *
1.14 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.15 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.16 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.17 + * version 2 for more details (a copy is included in the LICENSE file that
1.18 + * accompanied this code).
1.19 + *
1.20 + * You should have received a copy of the GNU General Public License version
1.21 + * 2 along with this work; if not, write to the Free Software Foundation,
1.22 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.23 + *
1.24 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.25 + * or visit www.oracle.com if you need additional information or have any
1.26 + * questions.
1.27 + */
1.28 +
1.29 +package java.util;
1.30 +
1.31 +import java.lang.*;
1.32 +
1.33 +/**
1.34 + * The string tokenizer class allows an application to break a
1.35 + * string into tokens. The tokenization method is much simpler than
1.36 + * the one used by the <code>StreamTokenizer</code> class. The
1.37 + * <code>StringTokenizer</code> methods do not distinguish among
1.38 + * identifiers, numbers, and quoted strings, nor do they recognize
1.39 + * and skip comments.
1.40 + * <p>
1.41 + * The set of delimiters (the characters that separate tokens) may
1.42 + * be specified either at creation time or on a per-token basis.
1.43 + * <p>
1.44 + * An instance of <code>StringTokenizer</code> behaves in one of two
1.45 + * ways, depending on whether it was created with the
1.46 + * <code>returnDelims</code> flag having the value <code>true</code>
1.47 + * or <code>false</code>:
1.48 + * <ul>
1.49 + * <li>If the flag is <code>false</code>, delimiter characters serve to
1.50 + * separate tokens. A token is a maximal sequence of consecutive
1.51 + * characters that are not delimiters.
1.52 + * <li>If the flag is <code>true</code>, delimiter characters are themselves
1.53 + * considered to be tokens. A token is thus either one delimiter
1.54 + * character, or a maximal sequence of consecutive characters that are
1.55 + * not delimiters.
1.56 + * </ul><p>
1.57 + * A <tt>StringTokenizer</tt> object internally maintains a current
1.58 + * position within the string to be tokenized. Some operations advance this
1.59 + * current position past the characters processed.<p>
1.60 + * A token is returned by taking a substring of the string that was used to
1.61 + * create the <tt>StringTokenizer</tt> object.
1.62 + * <p>
1.63 + * The following is one example of the use of the tokenizer. The code:
1.64 + * <blockquote><pre>
1.65 + * StringTokenizer st = new StringTokenizer("this is a test");
1.66 + * while (st.hasMoreTokens()) {
1.67 + * System.out.println(st.nextToken());
1.68 + * }
1.69 + * </pre></blockquote>
1.70 + * <p>
1.71 + * prints the following output:
1.72 + * <blockquote><pre>
1.73 + * this
1.74 + * is
1.75 + * a
1.76 + * test
1.77 + * </pre></blockquote>
1.78 + *
1.79 + * <p>
1.80 + * <tt>StringTokenizer</tt> is a legacy class that is retained for
1.81 + * compatibility reasons although its use is discouraged in new code. It is
1.82 + * recommended that anyone seeking this functionality use the <tt>split</tt>
1.83 + * method of <tt>String</tt> or the java.util.regex package instead.
1.84 + * <p>
1.85 + * The following example illustrates how the <tt>String.split</tt>
1.86 + * method can be used to break up a string into its basic tokens:
1.87 + * <blockquote><pre>
1.88 + * String[] result = "this is a test".split("\\s");
1.89 + * for (int x=0; x<result.length; x++)
1.90 + * System.out.println(result[x]);
1.91 + * </pre></blockquote>
1.92 + * <p>
1.93 + * prints the following output:
1.94 + * <blockquote><pre>
1.95 + * this
1.96 + * is
1.97 + * a
1.98 + * test
1.99 + * </pre></blockquote>
1.100 + *
1.101 + * @author unascribed
1.102 + * @see java.io.StreamTokenizer
1.103 + * @since JDK1.0
1.104 + */
1.105 +public
1.106 +class StringTokenizer implements Enumeration<Object> {
1.107 + private int currentPosition;
1.108 + private int newPosition;
1.109 + private int maxPosition;
1.110 + private String str;
1.111 + private String delimiters;
1.112 + private boolean retDelims;
1.113 + private boolean delimsChanged;
1.114 +
1.115 + /**
1.116 + * maxDelimCodePoint stores the value of the delimiter character with the
1.117 + * highest value. It is used to optimize the detection of delimiter
1.118 + * characters.
1.119 + *
1.120 + * It is unlikely to provide any optimization benefit in the
1.121 + * hasSurrogates case because most string characters will be
1.122 + * smaller than the limit, but we keep it so that the two code
1.123 + * paths remain similar.
1.124 + */
1.125 + private int maxDelimCodePoint;
1.126 +
1.127 + /**
1.128 + * If delimiters include any surrogates (including surrogate
1.129 + * pairs), hasSurrogates is true and the tokenizer uses the
1.130 + * different code path. This is because String.indexOf(int)
1.131 + * doesn't handle unpaired surrogates as a single character.
1.132 + */
1.133 + private boolean hasSurrogates = false;
1.134 +
1.135 + /**
1.136 + * When hasSurrogates is true, delimiters are converted to code
1.137 + * points and isDelimiter(int) is used to determine if the given
1.138 + * codepoint is a delimiter.
1.139 + */
1.140 + private int[] delimiterCodePoints;
1.141 +
1.142 + /**
1.143 + * Set maxDelimCodePoint to the highest char in the delimiter set.
1.144 + */
1.145 + private void setMaxDelimCodePoint() {
1.146 + if (delimiters == null) {
1.147 + maxDelimCodePoint = 0;
1.148 + return;
1.149 + }
1.150 +
1.151 + int m = 0;
1.152 + int c;
1.153 + int count = 0;
1.154 + for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) {
1.155 + c = delimiters.charAt(i);
1.156 + if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) {
1.157 + c = delimiters.codePointAt(i);
1.158 + hasSurrogates = true;
1.159 + }
1.160 + if (m < c)
1.161 + m = c;
1.162 + count++;
1.163 + }
1.164 + maxDelimCodePoint = m;
1.165 +
1.166 + if (hasSurrogates) {
1.167 + delimiterCodePoints = new int[count];
1.168 + for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) {
1.169 + c = delimiters.codePointAt(j);
1.170 + delimiterCodePoints[i] = c;
1.171 + }
1.172 + }
1.173 + }
1.174 +
1.175 + /**
1.176 + * Constructs a string tokenizer for the specified string. All
1.177 + * characters in the <code>delim</code> argument are the delimiters
1.178 + * for separating tokens.
1.179 + * <p>
1.180 + * If the <code>returnDelims</code> flag is <code>true</code>, then
1.181 + * the delimiter characters are also returned as tokens. Each
1.182 + * delimiter is returned as a string of length one. If the flag is
1.183 + * <code>false</code>, the delimiter characters are skipped and only
1.184 + * serve as separators between tokens.
1.185 + * <p>
1.186 + * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
1.187 + * not throw an exception. However, trying to invoke other methods on the
1.188 + * resulting <tt>StringTokenizer</tt> may result in a
1.189 + * <tt>NullPointerException</tt>.
1.190 + *
1.191 + * @param str a string to be parsed.
1.192 + * @param delim the delimiters.
1.193 + * @param returnDelims flag indicating whether to return the delimiters
1.194 + * as tokens.
1.195 + * @exception NullPointerException if str is <CODE>null</CODE>
1.196 + */
1.197 + public StringTokenizer(String str, String delim, boolean returnDelims) {
1.198 + currentPosition = 0;
1.199 + newPosition = -1;
1.200 + delimsChanged = false;
1.201 + this.str = str;
1.202 + maxPosition = str.length();
1.203 + delimiters = delim;
1.204 + retDelims = returnDelims;
1.205 + setMaxDelimCodePoint();
1.206 + }
1.207 +
1.208 + /**
1.209 + * Constructs a string tokenizer for the specified string. The
1.210 + * characters in the <code>delim</code> argument are the delimiters
1.211 + * for separating tokens. Delimiter characters themselves will not
1.212 + * be treated as tokens.
1.213 + * <p>
1.214 + * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
1.215 + * not throw an exception. However, trying to invoke other methods on the
1.216 + * resulting <tt>StringTokenizer</tt> may result in a
1.217 + * <tt>NullPointerException</tt>.
1.218 + *
1.219 + * @param str a string to be parsed.
1.220 + * @param delim the delimiters.
1.221 + * @exception NullPointerException if str is <CODE>null</CODE>
1.222 + */
1.223 + public StringTokenizer(String str, String delim) {
1.224 + this(str, delim, false);
1.225 + }
1.226 +
1.227 + /**
1.228 + * Constructs a string tokenizer for the specified string. The
1.229 + * tokenizer uses the default delimiter set, which is
1.230 + * <code>" \t\n\r\f"</code>: the space character,
1.231 + * the tab character, the newline character, the carriage-return character,
1.232 + * and the form-feed character. Delimiter characters themselves will
1.233 + * not be treated as tokens.
1.234 + *
1.235 + * @param str a string to be parsed.
1.236 + * @exception NullPointerException if str is <CODE>null</CODE>
1.237 + */
1.238 + public StringTokenizer(String str) {
1.239 + this(str, " \t\n\r\f", false);
1.240 + }
1.241 +
1.242 + /**
1.243 + * Skips delimiters starting from the specified position. If retDelims
1.244 + * is false, returns the index of the first non-delimiter character at or
1.245 + * after startPos. If retDelims is true, startPos is returned.
1.246 + */
1.247 + private int skipDelimiters(int startPos) {
1.248 + if (delimiters == null)
1.249 + throw new NullPointerException();
1.250 +
1.251 + int position = startPos;
1.252 + while (!retDelims && position < maxPosition) {
1.253 + if (!hasSurrogates) {
1.254 + char c = str.charAt(position);
1.255 + if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0))
1.256 + break;
1.257 + position++;
1.258 + } else {
1.259 + int c = str.codePointAt(position);
1.260 + if ((c > maxDelimCodePoint) || !isDelimiter(c)) {
1.261 + break;
1.262 + }
1.263 + position += Character.charCount(c);
1.264 + }
1.265 + }
1.266 + return position;
1.267 + }
1.268 +
1.269 + /**
1.270 + * Skips ahead from startPos and returns the index of the next delimiter
1.271 + * character encountered, or maxPosition if no such delimiter is found.
1.272 + */
1.273 + private int scanToken(int startPos) {
1.274 + int position = startPos;
1.275 + while (position < maxPosition) {
1.276 + if (!hasSurrogates) {
1.277 + char c = str.charAt(position);
1.278 + if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
1.279 + break;
1.280 + position++;
1.281 + } else {
1.282 + int c = str.codePointAt(position);
1.283 + if ((c <= maxDelimCodePoint) && isDelimiter(c))
1.284 + break;
1.285 + position += Character.charCount(c);
1.286 + }
1.287 + }
1.288 + if (retDelims && (startPos == position)) {
1.289 + if (!hasSurrogates) {
1.290 + char c = str.charAt(position);
1.291 + if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
1.292 + position++;
1.293 + } else {
1.294 + int c = str.codePointAt(position);
1.295 + if ((c <= maxDelimCodePoint) && isDelimiter(c))
1.296 + position += Character.charCount(c);
1.297 + }
1.298 + }
1.299 + return position;
1.300 + }
1.301 +
1.302 + private boolean isDelimiter(int codePoint) {
1.303 + for (int i = 0; i < delimiterCodePoints.length; i++) {
1.304 + if (delimiterCodePoints[i] == codePoint) {
1.305 + return true;
1.306 + }
1.307 + }
1.308 + return false;
1.309 + }
1.310 +
1.311 + /**
1.312 + * Tests if there are more tokens available from this tokenizer's string.
1.313 + * If this method returns <tt>true</tt>, then a subsequent call to
1.314 + * <tt>nextToken</tt> with no argument will successfully return a token.
1.315 + *
1.316 + * @return <code>true</code> if and only if there is at least one token
1.317 + * in the string after the current position; <code>false</code>
1.318 + * otherwise.
1.319 + */
1.320 + public boolean hasMoreTokens() {
1.321 + /*
1.322 + * Temporarily store this position and use it in the following
1.323 + * nextToken() method only if the delimiters haven't been changed in
1.324 + * that nextToken() invocation.
1.325 + */
1.326 + newPosition = skipDelimiters(currentPosition);
1.327 + return (newPosition < maxPosition);
1.328 + }
1.329 +
1.330 + /**
1.331 + * Returns the next token from this string tokenizer.
1.332 + *
1.333 + * @return the next token from this string tokenizer.
1.334 + * @exception NoSuchElementException if there are no more tokens in this
1.335 + * tokenizer's string.
1.336 + */
1.337 + public String nextToken() {
1.338 + /*
1.339 + * If next position already computed in hasMoreElements() and
1.340 + * delimiters have changed between the computation and this invocation,
1.341 + * then use the computed value.
1.342 + */
1.343 +
1.344 + currentPosition = (newPosition >= 0 && !delimsChanged) ?
1.345 + newPosition : skipDelimiters(currentPosition);
1.346 +
1.347 + /* Reset these anyway */
1.348 + delimsChanged = false;
1.349 + newPosition = -1;
1.350 +
1.351 + if (currentPosition >= maxPosition)
1.352 + throw new NoSuchElementException();
1.353 + int start = currentPosition;
1.354 + currentPosition = scanToken(currentPosition);
1.355 + return str.substring(start, currentPosition);
1.356 + }
1.357 +
1.358 + /**
1.359 + * Returns the next token in this string tokenizer's string. First,
1.360 + * the set of characters considered to be delimiters by this
1.361 + * <tt>StringTokenizer</tt> object is changed to be the characters in
1.362 + * the string <tt>delim</tt>. Then the next token in the string
1.363 + * after the current position is returned. The current position is
1.364 + * advanced beyond the recognized token. The new delimiter set
1.365 + * remains the default after this call.
1.366 + *
1.367 + * @param delim the new delimiters.
1.368 + * @return the next token, after switching to the new delimiter set.
1.369 + * @exception NoSuchElementException if there are no more tokens in this
1.370 + * tokenizer's string.
1.371 + * @exception NullPointerException if delim is <CODE>null</CODE>
1.372 + */
1.373 + public String nextToken(String delim) {
1.374 + delimiters = delim;
1.375 +
1.376 + /* delimiter string specified, so set the appropriate flag. */
1.377 + delimsChanged = true;
1.378 +
1.379 + setMaxDelimCodePoint();
1.380 + return nextToken();
1.381 + }
1.382 +
1.383 + /**
1.384 + * Returns the same value as the <code>hasMoreTokens</code>
1.385 + * method. It exists so that this class can implement the
1.386 + * <code>Enumeration</code> interface.
1.387 + *
1.388 + * @return <code>true</code> if there are more tokens;
1.389 + * <code>false</code> otherwise.
1.390 + * @see java.util.Enumeration
1.391 + * @see java.util.StringTokenizer#hasMoreTokens()
1.392 + */
1.393 + public boolean hasMoreElements() {
1.394 + return hasMoreTokens();
1.395 + }
1.396 +
1.397 + /**
1.398 + * Returns the same value as the <code>nextToken</code> method,
1.399 + * except that its declared return value is <code>Object</code> rather than
1.400 + * <code>String</code>. It exists so that this class can implement the
1.401 + * <code>Enumeration</code> interface.
1.402 + *
1.403 + * @return the next token in the string.
1.404 + * @exception NoSuchElementException if there are no more tokens in this
1.405 + * tokenizer's string.
1.406 + * @see java.util.Enumeration
1.407 + * @see java.util.StringTokenizer#nextToken()
1.408 + */
1.409 + public Object nextElement() {
1.410 + return nextToken();
1.411 + }
1.412 +
1.413 + /**
1.414 + * Calculates the number of times that this tokenizer's
1.415 + * <code>nextToken</code> method can be called before it generates an
1.416 + * exception. The current position is not advanced.
1.417 + *
1.418 + * @return the number of tokens remaining in the string using the current
1.419 + * delimiter set.
1.420 + * @see java.util.StringTokenizer#nextToken()
1.421 + */
1.422 + public int countTokens() {
1.423 + int count = 0;
1.424 + int currpos = currentPosition;
1.425 + while (currpos < maxPosition) {
1.426 + currpos = skipDelimiters(currpos);
1.427 + if (currpos >= maxPosition)
1.428 + break;
1.429 + currpos = scanToken(currpos);
1.430 + count++;
1.431 + }
1.432 + return count;
1.433 + }
1.434 +}