package httpfs
import (
"context"
"fmt"
"html/template"
"io"
"mime"
"net/http"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/c4pt0r/agfs/agfs-server/pkg/filesystem"
"github.com/c4pt0r/agfs/agfs-server/pkg/plugin"
"github.com/c4pt0r/agfs/agfs-server/pkg/plugin/config"
log "github.com/sirupsen/logrus"
)
const (
PluginName = "httpfs"
)
func getContentType(filename string) string {
baseName := filepath.Base(filename)
baseNameUpper := strings.ToUpper(baseName)
if baseNameUpper == "README" ||
strings.HasPrefix(baseNameUpper, "README.") {
return "text/plain; charset=utf-8"
}
ext := strings.ToLower(filepath.Ext(filename))
textTypes := map[string]string{
".txt": "text/plain; charset=utf-8",
".md": "text/markdown; charset=utf-8",
".markdown": "text/markdown; charset=utf-8",
".json": "application/json; charset=utf-8",
".xml": "application/xml; charset=utf-8",
".html": "text/html; charset=utf-8",
".htm": "text/html; charset=utf-8",
".css": "text/css; charset=utf-8",
".js": "application/javascript; charset=utf-8",
".yaml": "text/yaml; charset=utf-8",
".yml": "text/yaml; charset=utf-8",
".log": "text/plain; charset=utf-8",
".csv": "text/csv; charset=utf-8",
".sh": "text/x-shellscript; charset=utf-8",
".py": "text/x-python; charset=utf-8",
".go": "text/x-go; charset=utf-8",
".c": "text/x-c; charset=utf-8",
".cpp": "text/x-c++; charset=utf-8",
".h": "text/x-c; charset=utf-8",
".java": "text/x-java; charset=utf-8",
".rs": "text/x-rust; charset=utf-8",
".sql": "text/x-sql; charset=utf-8",
}
imageTypes := map[string]string{
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".gif": "image/gif",
".webp": "image/webp",
".svg": "image/svg+xml",
".ico": "image/x-icon",
".bmp": "image/bmp",
}
videoTypes := map[string]string{
".mp4": "video/mp4",
".webm": "video/webm",
".ogg": "video/ogg",
".avi": "video/x-msvideo",
".mov": "video/quicktime",
}
audioTypes := map[string]string{
".mp3": "audio/mpeg",
".wav": "audio/wav",
".ogg": "audio/ogg",
".m4a": "audio/mp4",
".flac": "audio/flac",
}
if ext == ".pdf" {
return "application/pdf"
}
if ct, ok := textTypes[ext]; ok {
return ct
}
if ct, ok := imageTypes[ext]; ok {
return ct
}
if ct, ok := videoTypes[ext]; ok {
return ct
}
if ct, ok := audioTypes[ext]; ok {
return ct
}
if ct := mime.TypeByExtension(ext); ct != "" {
return ct
}
return "application/octet-stream"
}
type HTTPFS struct {
agfsPath string
httpHost string
httpPort string
statusPath string
rootFS filesystem.FileSystem
mu sync.RWMutex
server *http.Server
pluginName string
startTime time.Time
}
func NewHTTPFS(agfsPath string, host string, port string, statusPath string, rootFS filesystem.FileSystem) (*HTTPFS, error) {
if agfsPath == "" {
return nil, fmt.Errorf("agfs_path is required")
}
if rootFS == nil {
return nil, fmt.Errorf("rootFS is required")
}
agfsPath = filesystem.NormalizePath(agfsPath)
statusPath = filesystem.NormalizePath(statusPath)
if host == "" {
host = "0.0.0.0"
}
if port == "" {
port = "8000"
}
fs := &HTTPFS{
agfsPath: agfsPath,
httpHost: host,
httpPort: port,
statusPath: statusPath,
rootFS: rootFS,
pluginName: PluginName,
startTime: time.Now(),
}
if err := fs.startHTTPServer(); err != nil {
return nil, fmt.Errorf("failed to start HTTP server: %w", err)
}
return fs, nil
}
func (fs *HTTPFS) resolveAGFSPath(urlPath string) string {
urlPath = filesystem.NormalizePath(urlPath)
if urlPath == "/" {
return fs.agfsPath
}
return path.Join(fs.agfsPath, urlPath)
}
func (fs *HTTPFS) startHTTPServer() error {
mux := http.NewServeMux()
mux.HandleFunc("/", fs.handleHTTPRequest)
addr := fs.httpHost + ":" + fs.httpPort
fs.server = &http.Server{
Addr: addr,
Handler: mux,
}
go func() {
log.Infof("[httpfs] Starting HTTP server on %s, serving AGFS path: %s", addr, fs.agfsPath)
log.Infof("[httpfs] HTTP server listening at http://%s:%s", fs.httpHost, fs.httpPort)
if err := fs.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Errorf("[httpfs] HTTP server error on %s: %v", addr, err)
} else if err == http.ErrServerClosed {
log.Infof("[httpfs] HTTP server on %s closed gracefully", addr)
}
}()
return nil
}
func (fs *HTTPFS) handleHTTPRequest(w http.ResponseWriter, r *http.Request) {
urlPath := r.URL.Path
pfsPath := fs.resolveAGFSPath(urlPath)
log.Infof("[httpfs:%s] %s %s (AGFS path: %s) from %s", fs.httpPort, r.Method, urlPath, pfsPath, r.RemoteAddr)
info, err := fs.rootFS.Stat(pfsPath)
if err != nil {
log.Warnf("[httpfs:%s] Not found: %s (AGFS: %s)", fs.httpPort, urlPath, pfsPath)
http.NotFound(w, r)
return
}
if info.IsDir {
fs.serveDirectory(w, r, pfsPath, urlPath)
return
}
fs.serveFile(w, r, pfsPath)
}
func (fs *HTTPFS) serveFile(w http.ResponseWriter, r *http.Request, pfsPath string) {
info, err := fs.rootFS.Stat(pfsPath)
if err != nil {
http.Error(w, "Failed to stat file", http.StatusInternalServerError)
log.Errorf("[httpfs:%s] Failed to stat file %s: %v", fs.httpPort, pfsPath, err)
return
}
contentType := getContentType(pfsPath)
log.Infof("[httpfs:%s] Serving file: %s (size: %d bytes, type: %s)", fs.httpPort, pfsPath, info.Size, contentType)
reader, err := fs.rootFS.Open(pfsPath)
if err != nil {
log.Debugf("[httpfs:%s] Open failed for %s, falling back to Read: %v", fs.httpPort, pfsPath, err)
data, err := fs.rootFS.Read(pfsPath, 0, -1)
if err != nil && err != io.EOF {
http.Error(w, "Failed to read file", http.StatusInternalServerError)
log.Errorf("[httpfs:%s] Failed to read file %s: %v", fs.httpPort, pfsPath, err)
return
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
w.Header().Set("Last-Modified", info.ModTime.Format(http.TimeFormat))
w.Write(data)
log.Infof("[httpfs:%s] Sent file: %s (%d bytes via Read)", fs.httpPort, pfsPath, len(data))
return
}
defer reader.Close()
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
w.Header().Set("Last-Modified", info.ModTime.Format(http.TimeFormat))
written, _ := io.Copy(w, reader)
log.Infof("[httpfs:%s] Sent file: %s (%d bytes via stream)", fs.httpPort, pfsPath, written)
}
func (fs *HTTPFS) serveDirectory(w http.ResponseWriter, r *http.Request, pfsPath string, urlPath string) {
entries, err := fs.rootFS.ReadDir(pfsPath)
if err != nil {
log.Errorf("[httpfs:%s] Failed to read directory %s: %v", fs.httpPort, pfsPath, err)
http.Error(w, "Failed to read directory", http.StatusInternalServerError)
return
}
log.Infof("[httpfs:%s] Serving directory: %s (%d entries)", fs.httpPort, pfsPath, len(entries))
sort.Slice(entries, func(i, j int) bool {
if entries[i].IsDir != entries[j].IsDir {
return entries[i].IsDir
}
return entries[i].Name < entries[j].Name
})
type FileEntry struct {
Name string
IsDir bool
Size int64
ModTime string
URL string
}
var files []FileEntry
for _, entry := range entries {
name := entry.Name
url := path.Join(urlPath, name)
if entry.IsDir {
name += "/"
url += "/"
}
files = append(files, FileEntry{
Name: name,
IsDir: entry.IsDir,
Size: entry.Size,
ModTime: entry.ModTime.Format("2006-01-02 15:04:05"),
URL: url,
})
}
tmpl := `<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Directory listing for {{.Path}}</title>
<style>
body { font-family: monospace; margin: 20px; }
h1 { border-bottom: 1px solid #ccc; padding-bottom: 10px; }
table { border-collapse: collapse; width: 100%; }
th, td { text-align: left; padding: 8px; }
tr:hover { background-color: #f5f5f5; }
th { background-color: #e0e0e0; }
a { text-decoration: none; color: #0066cc; }
a:hover { text-decoration: underline; }
.size { text-align: right; }
.info { color: #666; font-style: italic; margin-top: 20px; }
</style>
</head>
<body>
<h1>Directory listing for {{.Path}}</h1>
<hr>
{{if .Parent}}
<p><a href="{{.Parent}}">↑ Parent Directory</a></p>
{{end}}
<table>
<thead>
<tr>
<th>Name</th>
<th class="size">Size</th>
<th>Modified</th>
</tr>
</thead>
<tbody>
{{range .Files}}
<tr>
<td><a href="{{.URL}}">{{.Name}}</a></td>
<td class="size">{{if .IsDir}}-{{else}}{{.Size}}{{end}}</td>
<td>{{.ModTime}}</td>
</tr>
{{end}}
</tbody>
</table>
<hr>
<p class="info">agfs httagfs server - serving: {{.PFSPath}}</p>
</body>
</html>`
t, err := template.New("directory").Parse(tmpl)
if err != nil {
http.Error(w, "Template error", http.StatusInternalServerError)
return
}
parent := ""
if urlPath != "/" {
cleanPath := strings.TrimSuffix(urlPath, "/")
parent = path.Dir(cleanPath)
if parent != "/" && !strings.HasSuffix(parent, "/") {
parent = parent + "/"
}
}
data := struct {
Path string
PFSPath string
Parent string
Files []FileEntry
}{
Path: urlPath,
PFSPath: pfsPath,
Parent: parent,
Files: files,
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
t.Execute(w, data)
}
func (fs *HTTPFS) Create(path string) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Mkdir(path string, perm uint32) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Remove(path string) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) RemoveAll(path string) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Read(path string, offset int64, size int64) ([]byte, error) {
if path == "/" || path == "" {
statusData := []byte(fs.getStatusInfo())
if offset >= int64(len(statusData)) {
return []byte{}, io.EOF
}
data := statusData[offset:]
if size > 0 && int64(len(data)) > size {
data = data[:size]
}
return data, nil
}
return nil, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Write(path string, data []byte, offset int64, flags filesystem.WriteFlag) (int64, error) {
return 0, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) ReadDir(path string) ([]filesystem.FileInfo, error) {
return nil, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Stat(path string) (*filesystem.FileInfo, error) {
if path == "/" || path == "" {
statusData := fs.getStatusInfo()
return &filesystem.FileInfo{
Name: "status",
Size: int64(len(statusData)),
Mode: 0444,
ModTime: fs.startTime,
IsDir: false,
Meta: filesystem.MetaData{
Name: "httpfs-status",
Type: "virtual",
},
}, nil
}
return nil, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Rename(oldPath, newPath string) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Chmod(path string, mode uint32) error {
return fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) Open(path string) (io.ReadCloser, error) {
return nil, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) OpenWrite(path string) (io.WriteCloser, error) {
return nil, fmt.Errorf("httagfs is read-only via filesystem interface, use HTTP to access files")
}
func (fs *HTTPFS) getStatusInfo() string {
fs.mu.RLock()
defer fs.mu.RUnlock()
uptime := time.Since(fs.startTime)
status := fmt.Sprintf(`HTTPFS Instance Status
======================
Virtual Path: %s
AGFS Source Path: %s
HTTP Host: %s
HTTP Port: %s
HTTP Endpoint: http://%s:%s
Server Status: Running
Start Time: %s
Uptime: %s
Access this HTTP server:
Browser: http://%s:%s/
CLI: curl http://%s:%s/
Serving content from AGFS path: %s
All files under %s are accessible via HTTP on %s:%s
`,
fs.statusPath,
fs.agfsPath,
fs.httpHost,
fs.httpPort,
fs.httpHost,
fs.httpPort,
fs.startTime.Format("2006-01-02 15:04:05"),
uptime.Round(time.Second).String(),
fs.httpHost,
fs.httpPort,
fs.httpHost,
fs.httpPort,
fs.agfsPath,
fs.agfsPath,
fs.httpHost,
fs.httpPort,
)
return status
}
func (fs *HTTPFS) Shutdown() error {
if fs.server != nil {
log.Infof("[httpfs:%s] Shutting down HTTP server...", fs.httpPort)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := fs.server.Shutdown(ctx)
if err != nil {
log.Errorf("[httpfs:%s] Error during shutdown: %v", fs.httpPort, err)
} else {
log.Infof("[httpfs:%s] HTTP server shutdown complete", fs.httpPort)
}
return err
}
return nil
}
type HTTPFSPlugin struct {
fs *HTTPFS
agfsPath string
httpHost string
httpPort string
statusPath string
rootFS filesystem.FileSystem
}
func NewHTTPFSPlugin() *HTTPFSPlugin {
return &HTTPFSPlugin{}
}
func (p *HTTPFSPlugin) Name() string {
return PluginName
}
func (p *HTTPFSPlugin) Validate(cfg map[string]interface{}) error {
allowedKeys := []string{"agfs_path", "host", "port", "mount_path"}
if err := config.ValidateOnlyKnownKeys(cfg, allowedKeys); err != nil {
return err
}
if _, err := config.RequireString(cfg, "agfs_path"); err != nil {
return err
}
for _, key := range []string{"host", "mount_path"} {
if err := config.ValidateStringType(cfg, key); err != nil {
return err
}
}
if val, exists := cfg["port"]; exists {
switch val.(type) {
case string, int, int64, float64:
default:
return fmt.Errorf("port must be a string or number")
}
}
return nil
}
func (p *HTTPFSPlugin) SetRootFS(rootFS filesystem.FileSystem) {
p.rootFS = rootFS
}
func (p *HTTPFSPlugin) Initialize(config map[string]interface{}) error {
pfsPath, ok := config["agfs_path"].(string)
if !ok || pfsPath == "" {
return fmt.Errorf("agfs_path is required in configuration")
}
p.agfsPath = pfsPath
httpHost := "0.0.0.0"
if host, ok := config["host"].(string); ok && host != "" {
httpHost = host
}
p.httpHost = httpHost
httpPort := "8000"
if port, ok := config["port"].(string); ok && port != "" {
httpPort = port
} else if portInt, ok := config["port"].(int); ok {
httpPort = fmt.Sprintf("%d", portInt)
} else if portFloat, ok := config["port"].(float64); ok {
httpPort = fmt.Sprintf("%d", int(portFloat))
}
p.httpPort = httpPort
statusPath := "/"
if mountPath, ok := config["mount_path"].(string); ok && mountPath != "" {
statusPath = mountPath
}
p.statusPath = statusPath
if p.rootFS != nil {
fs, err := NewHTTPFS(p.agfsPath, p.httpHost, p.httpPort, p.statusPath, p.rootFS)
if err != nil {
return fmt.Errorf("failed to initialize httpfs: %w", err)
}
p.fs = fs
log.Infof("[httpfs] Initialized with AGFS path: %s, HTTP server: http://%s:%s, Status path: %s", pfsPath, httpHost, httpPort, statusPath)
} else {
log.Infof("[httpfs] Configured to serve AGFS path: %s on HTTP %s:%s (will start after rootFS is available)", pfsPath, httpHost, httpPort)
}
return nil
}
func (p *HTTPFSPlugin) GetFileSystem() filesystem.FileSystem {
if p.fs == nil && p.rootFS != nil {
fs, err := NewHTTPFS(p.agfsPath, p.httpHost, p.httpPort, p.statusPath, p.rootFS)
if err != nil {
log.Errorf("[httpfs] Failed to initialize: %v", err)
return nil
}
p.fs = fs
}
return p.fs
}
func (p *HTTPFSPlugin) GetReadme() string {
readmeContent := fmt.Sprintf(`HTTPFS Plugin - HTTP File Server for AGFS Paths
This plugin serves a AGFS mount path over HTTP, similar to 'python3 -m http.server'.
Unlike serving local files, this exposes any AGFS filesystem (memfs, queuefs, s3fs, etc.) via HTTP.
FEATURES:
- Serve any AGFS path via HTTP (e.g., /memfs, /queuefs, /s3fs)
- Browse files and directories in web browser
- Download files via HTTP
- Pretty HTML directory listings
- Access AGFS virtual filesystems through HTTP
- Read-only HTTP access (modifications should be done through AGFS API)
- Support for dynamic mounting via AGFS Shell mount command
CONFIGURATION:
Basic configuration:
[plugins.httpfs]
enabled = true
path = "/httpfs" # This is just a placeholder, not used for serving
[plugins.httpfs.config]
agfs_path = "/memfs" # The AGFS path to serve (e.g., /memfs, /queuefs)
host = "0.0.0.0" # Optional, defaults to 0.0.0.0 (all interfaces)
port = "8000" # Optional, defaults to 8000
Example - Serve memfs:
[plugins.httpfs_mem]
enabled = true
path = "/httpfs_mem"
[plugins.httpfs_mem.config]
agfs_path = "/memfs"
host = "localhost"
port = "9000"
Example - Serve queuefs:
[plugins.httpfs_queue]
enabled = true
path = "/httpfs_queue"
[plugins.httpfs_queue.config]
agfs_path = "/queuefs"
port = "9001"
CURRENT CONFIGURATION:
AGFS Path: %s
HTTP Server: http://%s:%s
DYNAMIC MOUNTING:
You can dynamically mount httagfs at runtime using AGFS Shell:
# In AGFS Shell REPL:
> mount httagfs /httpfs-demo agfs_path=/memfs port=10000
plugin mounted
> mount httagfs /web agfs_path=/local host=localhost port=9000
plugin mounted
> mounts
httagfs on /httpfs-demo (plugin: httpfs, agfs_path=/memfs, port=10000)
httagfs on /web (plugin: httpfs, agfs_path=/local, host=localhost, port=9000)
> unmount /httpfs-demo
Unmounted plugin at /httpfs-demo
# Via command line:
agfs mount httagfs /httpfs-demo agfs_path=/memfs port=10000
agfs mount httagfs /web agfs_path=/local host=localhost port=9000
agfs unmount /httpfs-demo
# Via REST API:
curl -X POST http://localhost:8080/api/v1/mount \
-H "Content-Type: application/json" \
-d '{
"fstype": "httpfs",
"path": "/httpfs-demo",
"config": {
"agfs_path": "/memfs",
"host": "0.0.0.0",
"port": "10000"
}
}'
Dynamic mounting advantages:
- No server restart required
- Mount/unmount on demand
- Multiple instances with different configurations
- Flexible port and path selection
USAGE:
Via Web Browser:
Open: http://localhost:%s
Browse directories and download files from AGFS
Via curl:
# List directory
curl http://localhost:%s/
# Download file
curl http://localhost:%s/file.txt
# Access subdirectory
curl http://localhost:%s/subdir/
EXAMPLES:
# Serve memfs on port 9000
http://localhost:9000 -> shows contents of /memfs
# Serve queuefs on port 9001
http://localhost:9001 -> shows contents of /queuefs
# Access files in browser
Open http://localhost:%s in your browser
Click on files to download
Click on directories to browse
NOTES:
- The HTTP server starts automatically when the plugin is initialized
- Files are served with proper MIME types
- Directory listings are formatted as pretty HTML
- httagfs provides HTTP read-only access to AGFS paths
- To modify files, use the AGFS API directly
- Multiple httagfs instances can serve different AGFS paths on different ports
USE CASES:
- Expose in-memory files (memfs) via HTTP for easy access
- Browse queue contents (queuefs) in a web browser
- Share S3 files (s3fs) through a simple HTTP interface
- Provide web access to any AGFS filesystem
- Quick file sharing without setting up separate web servers
- Debug and inspect AGFS filesystems visually
ADVANTAGES:
- Works with any AGFS filesystem (not just local files)
- Simple HTTP interface for complex backends
- Multiple instances can serve different paths
- No data duplication - serves directly from AGFS
- Lightweight and fast
VERSION: 1.0.0
AUTHOR: AGFS Server
`, p.agfsPath, p.httpHost, p.httpPort, p.httpPort, p.httpPort, p.httpPort, p.httpPort, p.httpPort)
return readmeContent
}
func (p *HTTPFSPlugin) GetConfigParams() []plugin.ConfigParameter {
return []plugin.ConfigParameter{
{
Name: "agfs_path",
Type: "string",
Required: true,
Default: "",
Description: "AGFS path to serve over HTTP (e.g., /memfs, /queuefs)",
},
{
Name: "host",
Type: "string",
Required: false,
Default: "0.0.0.0",
Description: "HTTP server host address",
},
{
Name: "port",
Type: "string",
Required: false,
Default: "8000",
Description: "HTTP server port",
},
}
}
func (p *HTTPFSPlugin) Shutdown() error {
log.Infof("[httpfs] Plugin shutting down (port: %s, path: %s)", p.httpPort, p.agfsPath)
if p.fs != nil {
return p.fs.Shutdown()
}
return nil
}
var _ plugin.ServicePlugin = (*HTTPFSPlugin)(nil)
var _ filesystem.FileSystem = (*HTTPFS)(nil)