Python’s lib2to3 package (deprecated Python 3.11, removed Python 3.13) is the concrete-syntax-tree parser and fixer framework that powers the 2to3 command-line tool. from lib2to3 import pygram, pytree and from lib2to3.pgen2 import driver. Parse: d = driver.Driver(pygram.python_grammar, convert=pytree.convert); tree = d.parse_string(src). Tree nodes: pytree.Node (interior, has .children, .type) and pytree.Leaf (terminal, has .value, .lineno, .column). Navigate: node.parent, node.children, node.next_sibling, node.prev_sibling. Search: node.pre_order() — depth-first iterator. Patch: leaf.value = "new_value" then str(tree) to regenerate source. Fixer: subclass lib2to3.fixer_base.BaseFix; set pattern (pattern-matching expression) and implement transform(node, results). Apply fixers: from lib2to3.refactor import RefactoringTool; tool = RefactoringTool(["lib2to3.fixes.fix_print"]); tree = tool.refactor_string(src, "<string>"). Claude Code generates custom source transformers, Python 2 compatibility auditors, migration scripts, and legacy codebase analysis tools.
CLAUDE.md for lib2to3
## lib2to3 Stack
- Stdlib: from lib2to3 import pygram, pytree (3.11 deprecated, 3.13 removed)
- from lib2to3.pgen2 import driver
- from lib2to3.refactor import RefactoringTool
- Parse: d = driver.Driver(pygram.python_grammar, convert=pytree.convert)
- tree = d.parse_string(src + "\\n")
- Tree: pytree.Node (interior) / pytree.Leaf (terminal)
- node.children / leaf.value / node.pre_order()
- Patch: leaf.value = "new_text"; str(tree) → modified source
- Fixer: class F(BaseFix): pattern = "..."; transform(node, results)
- Apply: RefactoringTool(fixer_names).refactor_string(src, "<name>")lib2to3 Source Transformation Pipeline
# app/lib2to3util.py — parser, navigator, leaf finder, renamer, fixer, auditor
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import Any, Iterator
_LIB2TO3_AVAILABLE = False
try:
from lib2to3 import pygram, pytree
from lib2to3.pgen2 import driver as _driver
from lib2to3.refactor import RefactoringTool
import lib2to3.fixer_base as _fixer_base
_LIB2TO3_AVAILABLE = True
except ImportError:
pass
# ─────────────────────────────────────────────────────────────────────────────
# 1. Parsing helpers
# ─────────────────────────────────────────────────────────────────────────────
def parse_source(src: str) -> "pytree.Base | None":
"""
Parse Python source into a lib2to3 concrete syntax tree.
Returns None if lib2to3 is unavailable or parse fails.
Example:
tree = parse_source("x = 1 + 2\\nprint(x)\\n")
print(type(tree)) # <class 'lib2to3.pytree.Node'>
"""
if not _LIB2TO3_AVAILABLE:
return None
# Ensure the source ends with a newline (lib2to3 requires it)
if not src.endswith("\n"):
src += "\n"
d = _driver.Driver(
pygram.python_grammar,
convert=pytree.convert,
)
return d.parse_string(src)
def tree_to_source(tree: "pytree.Base") -> str:
"""
Reconstruct source code from a lib2to3 tree (preserves whitespace).
Example:
tree = parse_source("x = 1\\n")
src = tree_to_source(tree)
assert "x = 1" in src
"""
return str(tree)
# ─────────────────────────────────────────────────────────────────────────────
# 2. Tree navigation helpers
# ─────────────────────────────────────────────────────────────────────────────
def walk_leaves(tree: "pytree.Base") -> "Iterator[pytree.Leaf]":
"""
Iterate over all Leaf nodes in pre-order (depth-first).
Example:
for leaf in walk_leaves(tree):
print(leaf.value, leaf.lineno)
"""
if not _LIB2TO3_AVAILABLE:
return
for node in tree.pre_order():
if isinstance(node, pytree.Leaf):
yield node
def find_leaves_by_value(tree: "pytree.Base", value: str) -> "list[pytree.Leaf]":
"""
Return all Leaf nodes whose .value equals value.
Example:
prints = find_leaves_by_value(tree, "print")
"""
return [leaf for leaf in walk_leaves(tree) if leaf.value == value]
def find_leaves_by_type(tree: "pytree.Base", token_type: int) -> "list[pytree.Leaf]":
"""
Return all Leaf nodes of a given token type (from lib2to3.pgen2.token).
Example:
from lib2to3.pgen2 import token
strings = find_leaves_by_type(tree, token.STRING)
"""
return [leaf for leaf in walk_leaves(tree) if leaf.type == token_type]
# ─────────────────────────────────────────────────────────────────────────────
# 3. Symbol renamer
# ─────────────────────────────────────────────────────────────────────────────
def rename_symbol(src: str, old_name: str, new_name: str) -> str:
"""
Rename all occurrences of a bare name token in source.
Returns the modified source string.
Example:
new_src = rename_symbol("def foo(): foo()", "foo", "bar")
# "def bar(): bar()"
"""
if not _LIB2TO3_AVAILABLE:
return src
tree = parse_source(src)
if tree is None:
return src
for leaf in walk_leaves(tree):
if leaf.value == old_name:
leaf.value = new_name
return tree_to_source(tree)
# ─────────────────────────────────────────────────────────────────────────────
# 4. Python 2 compatibility auditor
# ─────────────────────────────────────────────────────────────────────────────
@dataclass
class Py2Issue:
lineno: int
kind: str # "print_stmt", "exec_stmt", "backtick", "long_suffix", etc.
context: str
def audit_py2_patterns(src: str) -> list[Py2Issue]:
"""
Scan source for common Python 2-only patterns using leaf inspection.
Returns a list of Py2Issue records.
Example:
issues = audit_py2_patterns("print 'hello'\\nexec code\\n")
for issue in issues:
print(issue.lineno, issue.kind)
"""
issues: list[Py2Issue] = []
if not _LIB2TO3_AVAILABLE:
# Fallback: regex scan
patterns = [
("print_stmt", r"^\s*print\s+[^(]"),
("exec_stmt", r"^\s*exec\s+[^(]"),
("backtick", r"`[^`]+`"),
("long_suffix", r"\b\d+[lL]\b"),
]
for lineno, line in enumerate(src.splitlines(), 1):
for kind, pat in patterns:
if re.search(pat, line):
issues.append(Py2Issue(lineno=lineno, kind=kind, context=line.strip()))
return issues
tree = parse_source(src)
if tree is None:
return issues
# Walk leaves for known Py2 tokens
for leaf in walk_leaves(tree):
context = leaf.value.strip()
# print as statement (leaf value is "print" but parent is print_stmt)
if leaf.value == "print" and leaf.parent is not None:
ptype = getattr(leaf.parent, "type", None)
if ptype is not None:
try:
from lib2to3.pygram import python_grammar
sym = python_grammar.symbol2number.get("print_stmt")
if sym and ptype == sym:
issues.append(Py2Issue(
lineno=leaf.lineno,
kind="print_stmt",
context=str(leaf.parent).strip(),
))
except Exception:
pass
# long integer literals: 1L, 0x1L
if re.match(r"^\d+[lL]$", leaf.value):
issues.append(Py2Issue(lineno=leaf.lineno,
kind="long_suffix", context=context))
# backtick repr
if leaf.value == "`":
issues.append(Py2Issue(lineno=leaf.lineno,
kind="backtick", context="`...`"))
return issues
# ─────────────────────────────────────────────────────────────────────────────
# 5. Apply built-in lib2to3 fixers
# ─────────────────────────────────────────────────────────────────────────────
_BUILTIN_FIXERS = [
"lib2to3.fixes.fix_print",
"lib2to3.fixes.fix_raise",
"lib2to3.fixes.fix_except",
"lib2to3.fixes.fix_exec",
"lib2to3.fixes.fix_has_key",
"lib2to3.fixes.fix_dict",
"lib2to3.fixes.fix_unicode",
"lib2to3.fixes.fix_basestring",
"lib2to3.fixes.fix_xrange",
"lib2to3.fixes.fix_reduce",
]
def apply_fixers(src: str,
fixers: list[str] | None = None,
write: bool = False) -> str:
"""
Apply lib2to3 fixers to a source string and return the modified source.
fixers: list of fixer module names (default: common Python 2→3 fixers).
Example:
py3 = apply_fixers("print 'hello'\\nxrange(10)\\n")
# "print('hello')\\nrange(10)\\n"
"""
if not _LIB2TO3_AVAILABLE:
return src
if fixers is None:
fixers = _BUILTIN_FIXERS
# Filter to available fixers
available: list[str] = []
for fixer in fixers:
try:
__import__(fixer)
available.append(fixer)
except ImportError:
pass
if not available:
return src
tool = RefactoringTool(available, options={"print_function": True})
try:
tree = tool.refactor_string(src, "<string>")
if tree is None:
return src
return str(tree)
except Exception:
return src
# ─────────────────────────────────────────────────────────────────────────────
# Demo
# ─────────────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
print("=== lib2to3 demo ===")
print(f" lib2to3 available: {_LIB2TO3_AVAILABLE}")
py2_src = """\
import sys
def greet(name):
print "Hello,", name
def legacy(d):
if d.has_key("x"):
print d.keys()
for k in d.iteritems():
pass
return reduce(lambda a, b: a + b, xrange(10))
greet("world")
"""
# ── parse + walk leaves ───────────────────────────────────────────────────
print("\n--- parse + walk leaves ---")
tree = parse_source(py2_src)
if tree is not None:
leaves = list(walk_leaves(tree))
print(f" leaf count: {len(leaves)}")
print(f" first 10 leaf values: {[l.value for l in leaves[:10]]}")
# ── rename_symbol ─────────────────────────────────────────────────────────
print("\n--- rename_symbol: greet → welcome ---")
renamed = rename_symbol(py2_src, "greet", "welcome")
for line in renamed.splitlines()[:6]:
print(f" {line}")
# ── audit_py2_patterns ────────────────────────────────────────────────────
print("\n--- audit_py2_patterns ---")
issues = audit_py2_patterns(py2_src)
if issues:
for issue in issues:
print(f" line {issue.lineno:3d} [{issue.kind}] {issue.context[:50]}")
else:
print(" (no issues found — try with actual Python 2 source)")
# ── apply_fixers ──────────────────────────────────────────────────────────
print("\n--- apply_fixers (print + xrange + has_key + reduce) ---")
fixed = apply_fixers(py2_src)
print(" original: print \"Hello,\", name")
for line in fixed.splitlines():
if "Hello" in line or "xrange" in line or "has_key" in line or "reduce" in line:
print(f" fixed: {line.strip()}")
print("\n=== done ===")For the ast stdlib alternative — ast.parse(src) produces an abstract syntax tree (AST) rather than a concrete syntax tree (CST) so whitespace is discarded, but ast.NodeTransformer provides a clean visit_* API to rename, replace, or remove any AST node — use ast + ast.unparse() (Python 3.9+) for semantic code analysis and transformation that does not need to preserve formatting; use lib2to3 when you need a CST that preserves comments, whitespace, and formatting. For the libcst (PyPI) alternative — libcst.parse_module(src) provides a fully-typed, whitespace-preserving CST with a transformer API identical to ast.NodeTransformer — use libcst for all new Python source transformation tooling and codemods; lib2to3 is deprecated in Python 3.11 and removed in 3.13 while libcst is actively maintained and supports Python 3.8–3.12 syntax. The Claude Skills 360 bundle includes lib2to3 skill sets covering parse_source()/tree_to_source() round-trip helpers, walk_leaves()/find_leaves_by_value()/find_leaves_by_type() tree visitors, rename_symbol() name patcher, audit_py2_patterns()/Py2Issue compatibility scanner, and apply_fixers() fixer runner. Start with the free tier to try source migration patterns and lib2to3 pipeline code generation.