stuaxo · August 17, 2025 03:19 · Sep 10, 2024 · Sep 10, 2024 · May 11, 2024 · Apr 18, 2024
diff --git a/dirtollm.py b/dirtollm.py
@@ -32,6 +32,9 @@ def get_file_content(
         if not (include_empty or content.strip()):
             return None, None
 
+        if not include_binaries:
+            if "\0" in content:
+                return None, None
         return content, None
     except UnicodeDecodeError as ude:
         if not include_binaries:
@@ -75,7 +78,7 @@ def process_path(
                     )
                     if file_output is not None:
                         output += f"#:{path}:\n"
-                        output += file_output.rstrip("\n") + "\n"
+                        output += file_output.rstrip("\n") + "\n\n"
                         file_count += 1
                     if error and exit_on_error:
                         raise FileProcessingError(f"Exiting due to error in file: {path}")

diff --git a/dirtollm.py b/dirtollm.py
@@ -1,79 +1,227 @@
 #!/usr/bin/env python3
 
-# Usage: python dirtollm.py [--dir /path/to/directory] [--glob "*.py"] [--prompt [Custom prompt]] [--exclude "*.pyc"] [--copy] [--list]
+# Usage: python dirtollm.py [files or glob patterns...] [options]
+# Example: python dirtollm.py "*.py" "*.txt" /path/to/specific/file.py --exclude "*.pyc" --copy --verbose -x --binaries
 
 import argparse
 import pathlib
 import fnmatch
-import shlex
+import sys
+import os
+from typing import List, Tuple, Optional
 
 try:
     import pyperclip
+    PYPERCLIP_AVAILABLE = True
 except ImportError:
-    pyperclip = None
+    PYPERCLIP_AVAILABLE = False
+    pyperclip = None  # Keep linter happy
 
+class FileProcessingError(Exception):
+    pass
 
-def append_file_content(output, path):
+def get_file_content(
+    path: pathlib.Path,
+    errors: str,
+    verbose: bool,
+    include_binaries: bool,
+    include_empty: bool,
+) -> Tuple[Optional[str], Optional[Exception]]:
     try:
-        content = path.read_text()
-    except UnicodeDecodeError:
-        #content = "Skipped (binary file)\n\n"
-        content = ""
+        content = path.read_text(errors=errors)
+        if not (include_empty or content.strip()):
+            return None, None
+
+        return content, None
+    except UnicodeDecodeError as ude:
+        if not include_binaries:
+            return None, None
+        error_msg = f"#:{path}: Binary file\n"
+        if verbose:
+            error_msg += f"UnicodeDecodeError details: {ude}\n"
+        return f"{error_msg}\n", ude
     except Exception as ex:
-        content = f"Skipped (error reading file: {ex})\n\n"
-    output += f"#:{path}:\n{content}\n\n"
-    return output
+        error_msg = f"#:{path}: Read error\n"
+        if verbose:
+            error_msg += f"Error details: {ex}\n"
+        return f"{error_msg}\n", ex
 
-def fn_match_multiple(file, *patterns):
+def fn_matches_multiple(file: str, patterns: List[str]) -> bool:
     return any(fnmatch.fnmatch(file, pattern) for pattern in patterns)
 
-def dirtollm(output, directory, globs, excludes, listing=False):
+def process_path(
+    path: pathlib.Path,
+    globs: List[str],
+    excludes: List[str],
+    listing: bool,
+    errors: str,
+    verbose: bool,
+    exit_on_error: bool,
+    include_binaries: bool,
+    include_empty: bool,
+) -> Tuple[str, int]:
+    output = ""
     file_count = 0
-    p = pathlib.Path(directory)
 
-    for child in p.iterdir():
-        if child.is_dir() and not fn_match_multiple(pathlib.Path(child).name, *excludes):
-            output, sub_file_count = dirtollm(output, child, globs, excludes, listing=listing)
-            file_count += sub_file_count
-
-    for glob_pattern in globs:
-        for child in p.glob(glob_pattern):
-            if child.is_file() and not fn_match_multiple(child, *excludes):
+    if path.is_file():
+        if not globs or fn_matches_multiple(path.name, globs):
+            if not fn_matches_multiple(path.name, excludes):
                 if listing:
-                    output += f"{child}\n"
+                    output += f"{path}\n"
+                    file_count += 1
                 else:
-                    output = append_file_content(output, child)
-                file_count += 1
+                    file_output, error = get_file_content(
+                        path, errors, verbose, include_binaries, include_empty
+                    )
+                    if file_output is not None:
+                        output += f"#:{path}:\n"
+                        output += file_output.rstrip("\n") + "\n"
+                        file_count += 1
+                    if error and exit_on_error:
+                        raise FileProcessingError(f"Exiting due to error in file: {path}")
+    elif path.is_dir():
+        for child in path.iterdir():
+            child_output, child_count = process_path(
+                child, globs, excludes, listing, errors, verbose,
+                exit_on_error, include_binaries, include_empty
+            )
+            output += child_output
+            file_count += child_count
 
     return output, file_count
 
+def dirtollm(
+    paths: List[pathlib.Path],
+    globs: List[str],
+    excludes: List[str],
+    listing: bool = False,
+    errors: str = "replace",
+    verbose: bool = False,
+    exit_on_error: bool = False,
+    include_binaries: bool = False,
+    include_empty: bool = False,
+) -> Tuple[str, int]:
+    output = ""
+    total_file_count = 0
+
+    for path in paths:
+        path_output, file_count = process_path(
+            path, globs, excludes, listing, errors, verbose,
+            exit_on_error, include_binaries, include_empty
+        )
+        output += path_output
+        total_file_count += file_count
+
+    return output, total_file_count
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Process files based on specified paths or glob patterns.",
+        epilog='Example: python dirtollm.py "*.py" "*.txt" /path/to/specific/file.py --exclude "*.pyc" --copy --verbose -x --binaries',
+    )
+    parser.add_argument("paths", nargs="*", help="Files, directories, or glob patterns to process")
+    parser.add_argument(
+        "--exclude", nargs="+", help="Glob patterns to exclude", default=[]
+    )
+    parser.add_argument(
+        "--prompt",
+        nargs="?",
+        const="File contents:",
+        help="Specify prompt text to output before the files",
+    )
+    parser.add_argument(
+        "--count",
+        action="store_true",
+        help="Display the count of files, bytes, and tokens processed",
+    )
+    parser.add_argument(
+        "--copy",
+        action="store_true",
+        help="Copy output to the clipboard instead of printing to stdout",
+    )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="List all files that match the patterns without showing their contents",
+    )
+    parser.add_argument(
+        "--errors",
+        choices=["strict", "ignore", "replace", "backslashreplace"],
+        default="replace",
+        help="Specify how encoding errors are handled (default: replace)",
+    )
+    parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Enable verbose output for errors"
+    )
+    parser.add_argument(
+        "-x",
+        "--exit-on-error",
+        action="store_true",
+        help="Exit on first error encountered",
+    )
+    parser.add_argument(
+        "--binaries", action="store_true", help="Include non unicode files"
+    )
+    parser.add_argument("--empty", action="store_true", help="Include empty files")
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Process some files.')
-    parser.add_argument('--dir', type=str, help='Directory to process', default=".")
-    parser.add_argument('--exclude', nargs='+', type=str, help='Glob pattern to exclude', default=[])
-    parser.add_argument('--glob', nargs='+', type=str, help='Glob pattern to match', default="*")
-    parser.add_argument('--prompt', nargs='?', type=str, const="Filenames followed by file content-:", default=None, help='Specify prompt text to output before the files.')
-    parser.add_argument('--count', action='store_true', help='Display the count of files, bytes, and tokens processed')
-    parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
-    parser.add_argument('--list', action='store_true', help='List all files that match the patterns')
     args = parser.parse_args()
 
-    output = ""
-    if args.prompt is not None:
-        output += args.prompt + "\n"
+    paths = []
+    globs = []
+
+    if not args.paths:
+        paths = [pathlib.Path(".")]
+        globs = ["*"]
+    else:
+        for path_or_glob in args.paths:
+            path = pathlib.Path(path_or_glob)
+            if path.exists():
+                paths.append(path.resolve())
+            else:
+                paths.append(pathlib.Path.cwd())
+                globs.append(path_or_glob)
+
+    try:
+        output, file_count = dirtollm(
+            paths,
+            globs,
+            args.exclude,
+            listing=args.list,
+            errors=args.errors,
+            verbose=args.verbose,
+            exit_on_error=args.exit_on_error,
+            include_binaries=args.binaries,
+            include_empty=args.empty,
+        )
+    except FileProcessingError as fpe:
+        print(f"Error: {fpe}", file=sys.stderr)
+        sys.exit(1)
+
+    if args.prompt:
+        output = f"{args.prompt}\n\n{output}"
 
-    output, file_count = dirtollm(output, args.dir, args.glob, args.exclude, listing=args.list)
-    outpit = output.rstrip("\n")
+    output = output.rstrip("\n")
+    byte_count = len(output.encode("utf-8"))
     token_count = len(output.split())
 
     if args.count:
-        print(f"Processed {file_count} files, {len(output)} bytes, and approximately {token_count} tokens.")
+        print(
+            f"Processed {file_count} files, {byte_count} bytes, ~{token_count} tokens."
+        )
     elif args.copy:
-        if pyperclip:
+        if PYPERCLIP_AVAILABLE:
             pyperclip.copy(output)
-            print(f"Copied {file_count} files, {len(output)} bytes, and approximately {token_count} tokens to the clipboard.")
+            print(
+                f"Copied to clipboard: {file_count} files, {byte_count} bytes, ~{token_count} tokens."
+            )
         else:
-            print("The --copy option requires the 'pyperclip' module. Please install it to use this functionality.")
+            print(
+                "Error: --copy requires pyperclip module. Falling back to stdout.",
+                file=sys.stderr,
+            )
+            print(output)
     else:
         print(output)
+
+if __name__ == "__main__":
+    main()
diff --git a/dirtogpt.py b/dirtogpt.py
@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-# Usage: python dirtogpt.py [--dir /path/to/directory] [--glob *.py] [--prompt [Custom prompt]] [--copy] [--exclude *.pyc]
-
-import argparse
-import pathlib
-import fnmatch
-
-try:
-    import pyperclip
-except ImportError:
-    pyperclip = None
-
-
-def append_file_content(output, path):
-    try:
-        content = path.read_text()
-    except UnicodeDecodeError:
-        #content = "Skipped (binary file)\n\n"
-        content = ""
-    except Exception as ex:
-        content = f"Skipped (error reading file: {ex})\n\n"
-    output += f"#:{path}:\n{content}\n\n"
-    return output
-
-
-def dirtogpt(output, directory, glob, exclude):
-    file_count = 0
-    p = pathlib.Path(directory)
-    for child in p.iterdir():
-        if child.is_dir():
-            output, sub_file_count = dirtogpt(output, child, glob, exclude)
-            file_count += sub_file_count
-    for child in p.glob(glob):
-        if child.is_file() and not fnmatch.fnmatch(str(child), exclude):
-            output = append_file_content(output, child)
-            file_count += 1
-    return output, file_count
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Process some files.')
-    parser.add_argument('--dir', type=str, help='Directory to process', default=".")
-    parser.add_argument('--exclude', type=str, help='Glob pattern to exclude', default="")
-    parser.add_argument('--glob', type=str, help='Glob pattern to match', default="*")
-    parser.add_argument('--prompt', nargs='?', const="Filenames followed by file content-:", default=None, help='Display a prompt before the files')
-    parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
-    args = parser.parse_args()
-
-    output = ""
-    if args.prompt is not None:
-        output += args.prompt + "\n"
-
-    output, file_count = dirtogpt(output, args.dir, args.glob, args.exclude)
-    token_count = len(output.split())
-
-    if args.copy:
-        if pyperclip:
-            pyperclip.copy(output)
-            print(f"Copied {file_count} files, {len(output)} bytes, and approximately {token_count} tokens to the clipboard.")
-        else:
-            print("The --copy option requires the 'pyperclip' module. Please install it to use this functionality.")
-    else:
-        print(output)

diff --git a/dirtollm.py b/dirtollm.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+# Usage: python dirtollm.py [--dir /path/to/directory] [--glob "*.py"] [--prompt [Custom prompt]] [--exclude "*.pyc"] [--copy] [--list]
+
+import argparse
+import pathlib
+import fnmatch
+import shlex
+
+try:
+    import pyperclip
+except ImportError:
+    pyperclip = None
+
+
+def append_file_content(output, path):
+    try:
+        content = path.read_text()
+    except UnicodeDecodeError:
+        #content = "Skipped (binary file)\n\n"
+        content = ""
+    except Exception as ex:
+        content = f"Skipped (error reading file: {ex})\n\n"
+    output += f"#:{path}:\n{content}\n\n"
+    return output
+
+def fn_match_multiple(file, *patterns):
+    return any(fnmatch.fnmatch(file, pattern) for pattern in patterns)
+
+def dirtollm(output, directory, globs, excludes, listing=False):
+    file_count = 0
+    p = pathlib.Path(directory)
+
+    for child in p.iterdir():
+        if child.is_dir() and not fn_match_multiple(pathlib.Path(child).name, *excludes):
+            output, sub_file_count = dirtollm(output, child, globs, excludes, listing=listing)
+            file_count += sub_file_count
+
+    for glob_pattern in globs:
+        for child in p.glob(glob_pattern):
+            if child.is_file() and not fn_match_multiple(child, *excludes):
+                if listing:
+                    output += f"{child}\n"
+                else:
+                    output = append_file_content(output, child)
+                file_count += 1
+
+    return output, file_count
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Process some files.')
+    parser.add_argument('--dir', type=str, help='Directory to process', default=".")
+    parser.add_argument('--exclude', nargs='+', type=str, help='Glob pattern to exclude', default=[])
+    parser.add_argument('--glob', nargs='+', type=str, help='Glob pattern to match', default="*")
+    parser.add_argument('--prompt', nargs='?', type=str, const="Filenames followed by file content-:", default=None, help='Specify prompt text to output before the files.')
+    parser.add_argument('--count', action='store_true', help='Display the count of files, bytes, and tokens processed')
+    parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
+    parser.add_argument('--list', action='store_true', help='List all files that match the patterns')
+    args = parser.parse_args()
+
+    output = ""
+    if args.prompt is not None:
+        output += args.prompt + "\n"
+
+    output, file_count = dirtollm(output, args.dir, args.glob, args.exclude, listing=args.list)
+    outpit = output.rstrip("\n")
+    token_count = len(output.split())
+
+    if args.count:
+        print(f"Processed {file_count} files, {len(output)} bytes, and approximately {token_count} tokens.")
+    elif args.copy:
+        if pyperclip:
+            pyperclip.copy(output)
+            print(f"Copied {file_count} files, {len(output)} bytes, and approximately {token_count} tokens to the clipboard.")
+        else:
+            print("The --copy option requires the 'pyperclip' module. Please install it to use this functionality.")
+    else:
+        print(output)
diff --git a/dirtogpt.py b/dirtogpt.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
-# Usage: python dirtogpt.py [--dir /path/to/directory] [--glob *.py] [--prompt [Custom prompt]] [--copy]
+# Usage: python dirtogpt.py [--dir /path/to/directory] [--glob *.py] [--prompt [Custom prompt]] [--copy] [--exclude *.pyc]
 
 import argparse
 import pathlib
+import fnmatch
 
 try:
     import pyperclip
@@ -12,36 +13,46 @@
 
 
 def append_file_content(output, path):
-    output += f"#:{path}:\n{path.read_text()}\n\n"
+    try:
+        content = path.read_text()
+    except UnicodeDecodeError:
+        #content = "Skipped (binary file)\n\n"
+        content = ""
+    except Exception as ex:
+        content = f"Skipped (error reading file: {ex})\n\n"
+    output += f"#:{path}:\n{content}\n\n"
     return output
 
 
-def dirtogpt(output, directory, glob):
+def dirtogpt(output, directory, glob, exclude):
     file_count = 0
     p = pathlib.Path(directory)
+    for child in p.iterdir():
+        if child.is_dir():
+            output, sub_file_count = dirtogpt(output, child, glob, exclude)
+            file_count += sub_file_count
     for child in p.glob(glob):
-        if child.is_file():
+        if child.is_file() and not fnmatch.fnmatch(str(child), exclude):
             output = append_file_content(output, child)
             file_count += 1
-        elif child.is_dir():
-            output, sub_file_count = dirtogpt(output, child, glob)
-            file_count += sub_file_count
     return output, file_count
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Process some files.')
     parser.add_argument('--dir', type=str, help='Directory to process', default=".")
-    parser.add_argument('--glob', type=str, help='Glob pattern to match', default="**/*")
+    parser.add_argument('--exclude', type=str, help='Glob pattern to exclude', default="")
+    parser.add_argument('--glob', type=str, help='Glob pattern to match', default="*")
     parser.add_argument('--prompt', nargs='?', const="Filenames followed by file content-:", default=None, help='Display a prompt before the files')
     parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
     args = parser.parse_args()
 
-    output, file_count = dirtogpt(output, args.dir, args.glob)
-    token_count = len(output.split())
+    output = ""
+    if args.prompt is not None:
+        output += args.prompt + "\n"
 
-    if args.prompt is not None:        
-        output = args.prompt + "\n" + output
+    output, file_count = dirtogpt(output, args.dir, args.glob, args.exclude)
+    token_count = len(output.split())
 
     if args.copy:
         if pyperclip:

diff --git a/dirtogpt.py b/dirtogpt.py
@@ -37,13 +37,12 @@ def dirtogpt(output, directory, glob):
     parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
     args = parser.parse_args()
 
-    output = ""
-    if args.prompt is not None:
-        output += args.prompt + "\n"
-
     output, file_count = dirtogpt(output, args.dir, args.glob)
     token_count = len(output.split())
 
+    if args.prompt is not None:        
+        output = args.prompt + "\n" + output
+
     if args.copy:
         if pyperclip:
             pyperclip.copy(output)

diff --git a/dirtogpt.py b/dirtogpt.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# Usage: python dirtogpt.py [--dir /path/to/directory] [--glob *.py] [--prompt [Custom prompt]] [--copy]
+
+import argparse
+import pathlib
+
+try:
+    import pyperclip
+except ImportError:
+    pyperclip = None
+
+
+def append_file_content(output, path):
+    output += f"#:{path}:\n{path.read_text()}\n\n"
+    return output
+
+
+def dirtogpt(output, directory, glob):
+    file_count = 0
+    p = pathlib.Path(directory)
+    for child in p.glob(glob):
+        if child.is_file():
+            output = append_file_content(output, child)
+            file_count += 1
+        elif child.is_dir():
+            output, sub_file_count = dirtogpt(output, child, glob)
+            file_count += sub_file_count
+    return output, file_count
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Process some files.')
+    parser.add_argument('--dir', type=str, help='Directory to process', default=".")
+    parser.add_argument('--glob', type=str, help='Glob pattern to match', default="**/*")
+    parser.add_argument('--prompt', nargs='?', const="Filenames followed by file content-:", default=None, help='Display a prompt before the files')
+    parser.add_argument('--copy', action='store_true', help='Copy output to the clipboard instead of stdout')
+    args = parser.parse_args()
+
+    output = ""
+    if args.prompt is not None:
+        output += args.prompt + "\n"
+
+    output, file_count = dirtogpt(output, args.dir, args.glob)
+    token_count = len(output.split())
+
+    if args.copy:
+        if pyperclip:
+            pyperclip.copy(output)
+            print(f"Copied {file_count} files, {len(output)} bytes, and approximately {token_count} tokens to the clipboard.")
+        else:
+            print("The --copy option requires the 'pyperclip' module. Please install it to use this functionality.")
+    else:
+        print(output)