feat: Add GitHub App authentication for git cloning and releases

cachew.hcl

@@ -1,7 +1,3 @@
-# strategy git {}
-# strategy docker {}
-# strategy hermit {}
-
 # Artifactory caching proxy strategy
 # artifactory "example.jfrog.io" {
 #   target = "https://example.jfrog.io"
@@ -12,6 +8,12 @@ log {
   level = "debug"
 }
 
+github-app {
+  app-id             = "${GITHUB_APP_ID}"
+  private-key-path   = "${GITHUB_APP_PRIVATE_KEY_PATH}"
+  installations-json = "${GITHUB_APP_INSTALLATIONS}"
+}
+
 git {
   mirror-root = "./state/git-mirrors"
   bundle-interval = "24h"

cmd/cachewd/main.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/block/cachew/internal/cache"
 	"github.com/block/cachew/internal/config"
+	"github.com/block/cachew/internal/githubapp"
 	"github.com/block/cachew/internal/httputil"
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
@@ -34,6 +35,7 @@ type GlobalConfig struct {
 	SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"`
 	LoggingConfig   logging.Config      `embed:"" hcl:"log,block" prefix:"log-"`
 	MetricsConfig   metrics.Config      `embed:"" hcl:"metrics,block" prefix:"metrics-"`
+	GithubAppConfig githubapp.Config    `embed:"" hcl:"github-app,block" prefix:"github-app-"`
 }
 
 var cli struct {
@@ -62,6 +64,8 @@ func main() {
 
 	scheduler := jobscheduler.New(ctx, cli.SchedulerConfig)
 
+	githubapp.SetShared(initializeGitHubApp(ctx, logger, cli.GithubAppConfig))
+
 	cr, sr := newRegistries(scheduler)
 
 	// Commands
@@ -127,6 +131,32 @@ func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry)
 	}
 }
 
+func initializeGitHubApp(ctx context.Context, logger *slog.Logger, cfg githubapp.Config) *githubapp.TokenManager {
+	if err := cfg.Initialize(logger); err != nil {
+		logger.WarnContext(ctx, "Failed to initialize GitHub App config",
+			slog.String("error", err.Error()))
+		return nil
+	}
+
+	if !cfg.IsConfigured() {
+		return nil
+	}
+
+	tm, err := githubapp.NewTokenManager(cfg, githubapp.DefaultTokenCacheConfig())
+	if err != nil {
+		logger.ErrorContext(ctx, "Failed to create GitHub App token manager",
+			slog.String("error", err.Error()))
+		return nil
+	}
+
+	logger.InfoContext(ctx, "Global GitHub App authentication enabled",
+		slog.String("app_id", cfg.AppID),
+		slog.String("private_key_path", cfg.PrivateKeyPath),
+		slog.Int("installations", len(cfg.Installations)))
+
+	return tm
+}
+
 func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfig *hcl.AST) (*http.ServeMux, error) {
 	mux := http.NewServeMux()
 
@@ -140,7 +170,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov
 		_, _ = w.Write([]byte("OK")) //nolint:errcheck
 	})
 
-	if err := config.Load(ctx, cr, sr, providersConfig, mux, parseEnvars()); err != nil {
+	if err := config.Load(ctx, cr, sr, providersConfig, mux, config.ParseEnvars()); err != nil {
 		return nil, fmt.Errorf("load config: %w", err)
 	}
 
@@ -171,13 +201,3 @@ func newServer(ctx context.Context, logger *slog.Logger, mux *http.ServeMux) *ht
 		},
 	}
 }
-
-func parseEnvars() map[string]string {
-	envars := map[string]string{}
-	for _, env := range os.Environ() {
-		if key, value, ok := strings.Cut(env, "="); ok {
-			envars[key] = value
-		}
-	}
-	return envars
-}

docker/Dockerfile

@@ -6,7 +6,7 @@ ARG TARGETARCH
 SHELL ["/bin/sh", "-o", "pipefail", "-c"]
 
 # Install runtime dependencies for git operations and TLS
-RUN apk add --no-cache ca-certificates git tzdata && \
+RUN apk add --no-cache ca-certificates git git-daemon tzdata && \
   addgroup -g 1000 cachew && \
   adduser -D -u 1000 -G cachew cachew
 

go.mod

@@ -5,6 +5,7 @@ go 1.25.5
 require (
 	github.com/alecthomas/hcl/v2 v2.5.0
 	github.com/alecthomas/kong v1.13.0
+	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/goproxy/goproxy v0.25.0
 	github.com/lmittmann/tint v1.1.2
 	github.com/minio/minio-go/v7 v7.0.97

go.sum

@@ -35,6 +35,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=

internal/config/kong.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"io"
+	"os"
 	"strings"
 
 	"github.com/alecthomas/hcl/v2"
@@ -14,9 +15,24 @@ func KongLoader[GlobalConfig any](r io.Reader) (kong.Resolver, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse HCL: %w", err)
 	}
+
+	// Expand environment variables in the AST
+	expandVars(ast, ParseEnvars())
+
 	return &kongResolver{flattenHCL(ast)}, nil
 }
 
+// ParseEnvars returns a map of all environment variables.
+func ParseEnvars() map[string]string {
+	envars := map[string]string{}
+	for _, env := range os.Environ() {
+		if key, value, ok := strings.Cut(env, "="); ok {
+			envars[key] = value
+		}
+	}
+	return envars
+}
+
 type kongResolver struct {
 	values map[string]any
 }

internal/gitclone/command.go

@@ -5,13 +5,23 @@ package gitclone
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"os/exec"
 	"strings"
 
 	"github.com/alecthomas/errors"
 )
 
-func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) {
+func gitCommand(ctx context.Context, url string, credentialProvider CredentialProvider, args ...string) (*exec.Cmd, error) {
+	modifiedURL := url
+	if credentialProvider != nil && strings.Contains(url, "github.com") {
+		token, err := credentialProvider.GetTokenForURL(ctx, url)
+		if err == nil && token != "" {
+			modifiedURL = injectTokenIntoURL(url, token)
+		}
+		// If error getting token, fall back to original URL (system credentials)
+	}
+
 	configArgs, err := getInsteadOfDisableArgsForURL(ctx, url)
 	if err != nil {
 		return nil, errors.Wrap(err, "get insteadOf disable args")
@@ -23,8 +33,34 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err
 	}
 	allArgs = append(allArgs, args...)
 
-	cmd := exec.CommandContext(ctx, "git", allArgs...)
-	return cmd, nil
+	// Replace URL in args if it was modified for authentication
+	if modifiedURL != url {
+		for i, arg := range allArgs {
+			if arg == url {
+				allArgs[i] = modifiedURL
+			}
+		}
+	}
+
+	return exec.CommandContext(ctx, "git", allArgs...), nil
+}
+
+// Converts https://github.com/org/repo to https://x-access-token:TOKEN@github.com/org/repo
+func injectTokenIntoURL(url, token string) string {
+	if token == "" {
+		return url
+	}
+
+	if strings.HasPrefix(url, "https://github.com/") {
+		return strings.Replace(url, "https://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1)
+	}
+
+	// Upgrade http to https when adding token for security
+	if strings.HasPrefix(url, "http://github.com/") {
+		return strings.Replace(url, "http://github.com/", fmt.Sprintf("https://x-access-token:%s@github.com/", token), 1)
+	}
+
+	return url
 }
 
 func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) {

internal/gitclone/command_test.go

@@ -45,7 +45,7 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) {
 func TestGitCommand(t *testing.T) {
 	ctx := context.Background()
 
-	cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version")
+	cmd, err := gitCommand(ctx, "https://github.com/user/repo", nil, "version")
 	assert.NoError(t, err)
 
 	assert.NotZero(t, cmd)
@@ -59,7 +59,7 @@ func TestGitCommand(t *testing.T) {
 func TestGitCommandWithEmptyURL(t *testing.T) {
 	ctx := context.Background()
 
-	cmd, err := gitCommand(ctx, "", "version")
+	cmd, err := gitCommand(ctx, "", nil, "version")
 	assert.NoError(t, err)
 
 	assert.NotZero(t, cmd)

internal/gitclone/manager.go

@@ -74,24 +74,31 @@ type Config struct {
 	GitConfig        GitTuningConfig
 }
 
+// CredentialProvider provides credentials for git operations.
+type CredentialProvider interface {
+	GetTokenForURL(ctx context.Context, url string) (string, error)
+}
+
 type Repository struct {
-	mu            sync.RWMutex
-	state         State
-	path          string
-	upstreamURL   string
-	lastFetch     time.Time
-	lastRefCheck  time.Time
-	refCheckValid bool
-	fetchSem      chan struct{}
+	mu                 sync.RWMutex
+	state              State
+	path               string
+	upstreamURL        string
+	lastFetch          time.Time
+	lastRefCheck       time.Time
+	refCheckValid      bool
+	fetchSem           chan struct{}
+	credentialProvider CredentialProvider
 }
 
 type Manager struct {
-	config   Config
-	clones   map[string]*Repository
-	clonesMu sync.RWMutex
+	config             Config
+	clones             map[string]*Repository
+	clonesMu           sync.RWMutex
+	credentialProvider CredentialProvider
 }
 
-func NewManager(_ context.Context, config Config) (*Manager, error) {
+func NewManager(_ context.Context, config Config, credentialProvider CredentialProvider) (*Manager, error) {
 	if config.RootDir == "" {
 		return nil, errors.New("RootDir is required")
 	}
@@ -101,8 +108,9 @@ func NewManager(_ context.Context, config Config) (*Manager, error) {
 	}
 
 	return &Manager{
-		config: config,
-		clones: make(map[string]*Repository),
+		config:             config,
+		clones:             make(map[string]*Repository),
+		credentialProvider: credentialProvider,
 	}, nil
 }
 
@@ -125,10 +133,11 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor
 	clonePath := m.clonePathForURL(upstreamURL)
 
 	repo = &Repository{
-		state:       StateEmpty,
-		path:        clonePath,
-		upstreamURL: upstreamURL,
-		fetchSem:    make(chan struct{}, 1),
+		state:              StateEmpty,
+		path:               clonePath,
+		upstreamURL:        upstreamURL,
+		fetchSem:           make(chan struct{}, 1),
+		credentialProvider: m.credentialProvider,
 	}
 
 	gitDir := filepath.Join(clonePath, ".git")
@@ -152,6 +161,10 @@ func (m *Manager) Config() Config {
 	return m.config
 }
 
+func (m *Manager) CredentialProvider() CredentialProvider {
+	return m.credentialProvider
+}
+
 func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 	var discovered []*Repository
 	err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error {
@@ -195,10 +208,11 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 		upstreamURL := "https://" + host + "/" + repoPath
 
 		repo := &Repository{
-			state:       StateReady,
-			path:        path,
-			upstreamURL: upstreamURL,
-			fetchSem:    make(chan struct{}, 1),
+			state:              StateReady,
+			path:               path,
+			upstreamURL:        upstreamURL,
+			fetchSem:           make(chan struct{}, 1),
+			credentialProvider: m.credentialProvider,
 		}
 		repo.fetchSem <- struct{}{}
 
@@ -304,7 +318,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error {
 		r.upstreamURL, r.path,
 	}
 
-	cmd, err := gitCommand(ctx, r.upstreamURL, args...)
+	cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, args...)
 	if err != nil {
 		return errors.Wrap(err, "create git command")
 	}
@@ -320,7 +334,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error {
 		return errors.Wrapf(err, "configure fetch refspec: %s", string(output))
 	}
 
-	cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path,
+	cmd, err = gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path,
 		"-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer),
 		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit),
 		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())),
@@ -357,7 +371,7 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error {
 	r.mu.Lock()
 
 	// #nosec G204 - r.path is controlled by us
-	cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path,
+	cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "-C", r.path,
 		"-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer),
 		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit),
 		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())),
@@ -447,7 +461,7 @@ func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error
 
 func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) {
 	// #nosec G204 - r.upstreamURL is controlled by us
-	cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL)
+	cmd, err := gitCommand(ctx, r.upstreamURL, r.credentialProvider, "ls-remote", r.upstreamURL)
 	if err != nil {
 		return nil, errors.Wrap(err, "create git command")
 	}