Pythonpythonaitransformersbatch-processingnlp
Batch Text Summarizer
A Python script that reads text files from a directory and generates concise summaries using a local transformer model.
from pathlib import Path
from transformers import pipeline
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
def summarize_files(directory: str, max_length: int = 130) -> dict[str, str]:
results: dict[str, str] = {}
for file_path in Path(directory).glob("*.txt"):
text = file_path.read_text(encoding="utf-8")
summary = summarizer(text, max_length=max_length, min_length=30, do_sample=False)
results[file_path.name] = summary[0]["summary_text"]
return results
if __name__ == "__main__":
summaries = summarize_files("./documents")
for name, summary in summaries.items():
print(f"{name}:\n{summary}\n")title: "Batch Text Summarizer" description: "A Python script that reads text files from a directory and generates concise summaries using a local transformer model." category: "Python" language: "python" tags: ["python", "ai", "transformers", "batch-processing", "nlp"] createdAt: "2024-11-20" updatedAt: "2024-12-05"
Overview
This script leverages the Hugging Face Transformers library to batch-process text files and generate concise summaries. It uses the BART-Large-CNN model, which is pre-trained for abstractive summarization tasks.
Code
from pathlib import Path
from transformers import pipeline
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
def summarize_files(directory: str, max_length: int = 130) -> dict[str, str]:
results: dict[str, str] = {}
for file_path in Path(directory).glob("*.txt"):
text = file_path.read_text(encoding="utf-8")
summary = summarizer(text, max_length=max_length, min_length=30, do_sample=False)
results[file_path.name] = summary[0]["summary_text"]
return results
if __name__ == "__main__":
summaries = summarize_files("./documents")
for name, summary in summaries.items():
print(f"{name}:\n{summary}\n")Usage
# Install dependencies
# pip install transformers torch
# Directory structure:
# ./documents/
# article1.txt
# article2.txt
# report.txt
from batch_summarizer import summarize_files
# Basic usage
summaries = summarize_files("./documents")
# Custom max length
summaries = summarize_files("./documents", max_length=200)
# Process results
for filename, summary in summaries.items():
print(f"File: {filename}")
print(f"Summary: {summary}")
print("---")Notes
- The BART-Large-CNN model requires ~1.6GB of disk space on first download
- Input text should be between 50 and 1024 tokens for best results
- For GPU acceleration, install PyTorch with CUDA support
- Consider chunking for documents longer than the model's max input length (1024 tokens)
- Add error handling for empty files or encoding issues in production