Gogoconcurrencyworker-poolchannelsproduction-ready

Concurrent Worker Pool

A generic worker pool pattern using goroutines and channels for bounded parallel processing with graceful shutdown.

package workerpool

import (
	"context"
	"sync"
	"sync/atomic"
)

type Job[T any, R any] struct {
	ID    int
	Input T
}

type Result[R any] struct {
	JobID int
	Value R
	Err   error
}

type Pool[T any, R any] struct {
	workers    int
	jobChan    chan Job[T, R]
	resultChan chan Result[R]
	handler    func(context.Context, T) (R, error)
	wg         sync.WaitGroup
	processed  int64
	failed     int64
}

func New[T any, R any](workers int, bufferSize int, handler func(context.Context, T) (R, error)) *Pool[T, R] {
	return &Pool[T, R]{
		workers:    workers,
		jobChan:    make(chan Job[T, R], bufferSize),
		resultChan: make(chan Result[R], bufferSize),
		handler:    handler,
	}
}

func (p *Pool[T, R]) Start(ctx context.Context) {
	for i := 0; i < p.workers; i++ {
		p.wg.Add(1)
		go p.worker(ctx)
	}

	// Close results channel when all workers finish
	go func() {
		p.wg.Wait()
		close(p.resultChan)
	}()
}

func (p *Pool[T, R]) worker(ctx context.Context) {
	defer p.wg.Done()
	for {
		select {
		case <-ctx.Done():
			return
		case job, ok := <-p.jobChan:
			if !ok {
				return
			}
			result, err := p.handler(ctx, job.Input)
			if err != nil {
				atomic.AddInt64(&p.failed, 1)
				p.resultChan <- Result[R]{JobID: job.ID, Err: err}
			} else {
				atomic.AddInt64(&p.processed, 1)
				p.resultChan <- Result[R]{JobID: job.ID, Value: result}
			}
		}
	}
}

func (p *Pool[T, R]) Submit(job Job[T, R]) {
	p.jobChan <- job
}

func (p *Pool[T, R]) Close() {
	close(p.jobChan)
}

func (p *Pool[T, R]) Results() <-chan Result[R] {
	return p.resultChan
}

func (p *Pool[T, R]) Stats() (processed, failed int64) {
	return atomic.LoadInt64(&p.processed), atomic.LoadInt64(&p.failed)
}

// Convenience function for simple map-style parallelism
func Map[T any, R any](ctx context.Context, items []T, workers int, fn func(context.Context, T) (R, error)) ([]Result[R], error) {
	pool := New[T, R](workers, len(items), fn)
	pool.Start(ctx)

	// Submit all jobs
	go func() {
		for i, item := range items {
			pool.Submit(Job[T, R]{ID: i, Input: item})
		}
		pool.Close()
	}()

	// Collect results
	results := make([]Result[R], 0, len(items))
	for result := range pool.Results() {
		results = append(results, result)
	}

	return results, nil
}

title: "Concurrent Worker Pool" description: "A generic worker pool pattern using goroutines and channels for bounded parallel processing with graceful shutdown." category: "Go" language: "go" tags: ["go", "concurrency", "worker-pool", "channels", "production-ready"] createdAt: "2025-01-20" updatedAt: "2025-02-10"

Overview

A generic worker pool using Go 1.18+ generics that distributes work across a fixed number of goroutines via channels. Supports context cancellation, bounded concurrency, and type-safe results without interface assertions.

Code

pool := workerpool.New[string, []byte](10, 100, fetchURL)
pool.Start(ctx)
pool.Submit(Job{ID: 1, Input: "https://..."})
pool.Close()
for result := range pool.Results() { ... }

Notes

  • Generics eliminate the need for type assertions on inputs and outputs
  • Buffered channels prevent goroutine leaks when producers are faster than workers
  • Context cancellation provides clean shutdown when deadlines are exceeded
  • Map is a convenience wrapper for simple parallel-map operations
  • Atomic counters track processed/failed counts without lock contention