StringTokenizer

// ── Basic StringTokenizer usage: ─────────────────────────────────────
String sentence = "The quick brown fox";

StringTokenizer st = new StringTokenizer(sentence);
// Default delimiter is " 	

" (whitespace characters)

System.out.println("Token count: " + st.countTokens()); // 4

while (st.hasMoreTokens()) {
    System.out.println(st.nextToken());
}
// The
// quick
// brown
// fox

// ── Custom delimiter: ─────────────────────────────────────────────────
String csv = "Alice,30,Engineer,London";
StringTokenizer csvSt = new StringTokenizer(csv, ",");

while (csvSt.hasMoreTokens()) {
    System.out.println(csvSt.nextToken());
}
// Alice
// 30
// Engineer
// London

// ── Multiple delimiter characters — any char in the string is a delimiter: ─
String mixed = "one:two;three:four;five";
StringTokenizer multiSt = new StringTokenizer(mixed, ":;");
// Both ':' and ';' are delimiters

while (multiSt.hasMoreTokens()) {
    System.out.print(multiSt.nextToken() + " ");
}
// one two three four five

// ── Consecutive delimiters — SKIPPED (no empty tokens): ───────────────
String withGaps = "one,,two,,three";
StringTokenizer gapSt = new StringTokenizer(withGaps, ",");
System.out.println("Tokens: " + gapSt.countTokens());  // 3 — not 5!

while (gapSt.hasMoreTokens()) {
    System.out.println("[" + gapSt.nextToken() + "]");
}
// [one]
// [two]
// [three]
// Empty strings between consecutive commas are SKIPPED.

// ── Compare with String.split() — different behaviour: ────────────────
String[] parts = "one,,two,,three".split(",");
System.out.println("Parts: " + parts.length);    // 5
for (String p : parts) System.out.println("[" + p + "]");
// [one]
// []      ← empty token for the empty field
// [two]
// []      ← empty token for the empty field
// [three]

// ── Constructor 1: default whitespace delimiters: ────────────────────
StringTokenizer st1 = new StringTokenizer("  hello   world  ");
System.out.println(st1.countTokens());  // 2 — leading/trailing spaces skipped
while (st1.hasMoreTokens()) System.out.print(st1.nextToken() + "|");
// hello|world|

// ── Constructor 2: custom delimiter string: ───────────────────────────
StringTokenizer st2 = new StringTokenizer("a=b&c=d", "=&");
// Every char in "=&" is a delimiter: '=' and '&'
while (st2.hasMoreTokens()) System.out.print(st2.nextToken() + " ");
// a b c d

// ── Constructor 3: returnDelims=true — delimiters returned as tokens: ─
StringTokenizer st3 = new StringTokenizer("a+b*c", "+*", true);
while (st3.hasMoreTokens()) {
    String token = st3.nextToken();
    if (token.equals("+"))      System.out.print("[PLUS]");
    else if (token.equals("*")) System.out.print("[MULT]");
    else                         System.out.print(token);
}
// a[PLUS]b[MULT]c

// ── nextToken(delimiter) — change delimiter mid-stream: ───────────────
StringTokenizer dynamic = new StringTokenizer("key=value&next");
String key   = dynamic.nextToken("=");    // "key"   — up to '='
String value = dynamic.nextToken("&");    // "value" — up to '&'
String rest  = dynamic.nextToken();       // "next"  — remaining
System.out.printf("key=%s value=%s rest=%s%n", key, value, rest);

// ── Collecting all tokens into a List: ────────────────────────────────
StringTokenizer st4 = new StringTokenizer("one two three four five");
List<String> tokens = new ArrayList<>();
while (st4.hasMoreTokens()) {
    tokens.add(st4.nextToken());
}
System.out.println(tokens);  // [one, two, three, four, five]

// ── Three ways to split a string — comparison: ────────────────────────

String input = "Alice,30,Engineer,London";

// 1. StringTokenizer — fast, no empty tokens, legacy API:
StringTokenizer st = new StringTokenizer(input, ",");
List<String> tokenizerResult = new ArrayList<>();
while (st.hasMoreTokens()) tokenizerResult.add(st.nextToken());
System.out.println("Tokenizer: " + tokenizerResult);
// [Alice, 30, Engineer, London]

// 2. String.split() — regex, returns array, modern:
String[] splitResult = input.split(",");
System.out.println("split():   " + Arrays.toString(splitResult));
// [Alice, 30, Engineer, London]

// 3. Scanner — typed parsing, streaming:
Scanner sc = new Scanner(input).useDelimiter(",");
List<String> scannerResult = new ArrayList<>();
while (sc.hasNext()) scannerResult.add(sc.next());
System.out.println("Scanner:   " + scannerResult);
// [Alice, 30, Engineer, London]

// ── Empty token behaviour — critical difference: ──────────────────────
String withEmpties = "one,,three,,five";

// StringTokenizer — skips empty tokens:
StringTokenizer st2 = new StringTokenizer(withEmpties, ",");
System.out.println("Tokenizer count: " + st2.countTokens());  // 3

// String.split() — preserves empty tokens by default:
String[] parts = withEmpties.split(",");
System.out.println("split() count:   " + parts.length);       // 5
System.out.println(Arrays.toString(parts));  // [one, , three, , five]

// String.split() with limit=-1 — includes trailing empty: ─────────────
String trailing = "a,b,c,";
System.out.println(Arrays.toString(trailing.split(",")));       // [a, b, c]
System.out.println(Arrays.toString(trailing.split(",", -1)));   // [a, b, c, ]

// ── When to use each: ─────────────────────────────────────────────────
//
// StringTokenizer:
//   ✓ Simple delimiter chars (no regex)
//   ✓ Fastest for simple splits
//   ✓ Memory-efficient (lazy, no array allocation)
//   ✗ No empty tokens (wrong for CSV with empty fields)
//   ✗ No regex support
//   ✗ Legacy Enumeration API
//   → Use in competitive programming / legacy code only
//
// String.split(regex):
//   ✓ Regex support — powerful pattern matching
//   ✓ Returns array — easy iteration
//   ✓ Preserves empty tokens
//   ✗ Slower (regex compilation)
//   ✗ Returns all tokens at once
//   → Standard choice for most use cases
//
// Scanner.useDelimiter(regex):
//   ✓ Regex support
//   ✓ Lazy streaming — one token at a time
//   ✓ Type-parsing (nextInt, nextDouble, etc.)
//   ✓ Works with files, streams, strings
//   → Use when parsing typed data from any character source