From a00d948b9d861285a469f136f8476923111f640a Mon Sep 17 00:00:00 2001 From: root Date: Sat, 6 Jun 2026 17:12:29 +0000 Subject: [PATCH] Async deploy pipeline + port recycling + central config - Submit returns 202 with deploymentID; client polls GET /projects/:id. Bounded goroutine pool (8) with panic recovery so build/run can take minutes without HTTP timeouts. - Port allocator: reuses pooled Mongo client (was open-per-call), atomic InsertOne with unique index (was racy upsert), recycle pool via db.ReleasePort on delete (was permanently watermarked). - Fix DeleteProjectByContainerName field name: containername -> container_name (was silently no-op, leaving orphan project docs). - New internal/config package loads .env once at startup via sync.Once. Removed duplicate godotenv.Load from utils/jwt.go and utils/port_manager.go's init(). - FUTURE comments added for follow-up items (docker bind cleanup, double AuthorizePort, clone-path race, GH callback nil panic, hardcoded localhost redirect, Go Dockerfile template, containerName collision, signup validation). Co-Authored-By: Claude Opus 4.7 --- autoship-server/cmd/server/main.go | 26 +- autoship-server/internal/api/auth.go | 10 + autoship-server/internal/api/deploy.go | 146 +++++++++++ autoship-server/internal/api/projects.go | 177 +++++--------- autoship-server/internal/api/router.go | 1 + autoship-server/internal/config/config.go | 76 ++++++ autoship-server/internal/db/DeleteProjects.go | 2 +- autoship-server/internal/db/indexes.go | 32 +++ autoship-server/internal/db/ports.go | 40 ++++ autoship-server/internal/db/projects.go | 31 ++- autoship-server/internal/models/projects.go | 14 +- autoship-server/internal/services/clone.go | 5 + autoship-server/internal/services/dynamic.go | 16 ++ autoship-server/internal/utils/jwt.go | 10 +- .../internal/utils/port_manager.go | 226 +++++++++--------- 15 files changed, 549 insertions(+), 263 deletions(-) create mode 100644 autoship-server/internal/api/deploy.go create mode 100644 autoship-server/internal/config/config.go create mode 100644 autoship-server/internal/db/indexes.go create mode 100644 autoship-server/internal/db/ports.go diff --git a/autoship-server/cmd/server/main.go b/autoship-server/cmd/server/main.go index 8425497..16752f3 100644 --- a/autoship-server/cmd/server/main.go +++ b/autoship-server/cmd/server/main.go @@ -2,46 +2,40 @@ package main import ( "log" - "os" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/api" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/cloud" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/config" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/db" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/utils" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cors" - "github.com/joho/godotenv" ) func main() { - if err := godotenv.Load(); err != nil { - log.Println("No .env file found, using environment variables") + cfg, err := config.Load() + if err != nil { + log.Fatalf("config: %v", err) } - if err := cloud.Init(os.Getenv("CLOUD_PROVIDER")); err != nil { + if err := cloud.Init(cfg.CloudProvider); err != nil { log.Fatalf("Failed to initialize cloud provider: %v", err) } if err := utils.LoadEnv(); err != nil { log.Fatalf("Error loading JWT environment variables: %v", err) } - mongoURI := os.Getenv("MONGO_URI") - if mongoURI == "" { - log.Fatal("MONGO_URI is not set") - } - db.SetMongoURI(mongoURI) + db.SetMongoURI(cfg.MongoURI) db.Connect() defer db.Disconnect() - - port := os.Getenv("PORT") - if port == "" { - port = "3000" + if err := db.EnsurePortsIndex(cfg.MongoCollection); err != nil { + log.Fatalf("Failed to ensure ports unique index: %v", err) } app := fiber.New() app.Use(cors.New()) api.RegisterRoutes(app) - log.Printf("🚀 Server running on http://localhost:%s", port) - log.Fatal(app.Listen(":" + port)) + log.Printf("🚀 Server running on http://localhost:%s", cfg.ServerPort) + log.Fatal(app.Listen(":" + cfg.ServerPort)) } diff --git a/autoship-server/internal/api/auth.go b/autoship-server/internal/api/auth.go index 93f0a3a..45f6866 100644 --- a/autoship-server/internal/api/auth.go +++ b/autoship-server/internal/api/auth.go @@ -17,6 +17,11 @@ import ( ) // Signup handler +// +// FUTURE: no input validation — empty email or empty password parses +// successfully and creates an unusable user (bcrypt happily hashes ""). +// Add explicit non-empty + email-format checks before the existing-user +// lookup. func Signup(c *fiber.Ctx) error { var user models.User if err := c.BodyParser(&user); err != nil { @@ -123,6 +128,9 @@ func GitHubCallback(c *fiber.Ctx) error { // Extract email (or fallback) email, ok := userInfo["email"].(string) if !ok || email == "" { + // FUTURE: panics if userInfo["login"] is nil or non-string. Fiber + // catches it as 500 but the safe form is: + // if login, ok := userInfo["login"].(string); ok { email = login + "@github.com" } email = userInfo["login"].(string) + "@github.com" } @@ -162,6 +170,8 @@ func GitHubCallback(c *fiber.Ctx) error { // }) // Redirect to frontend dashboard with token + // FUTURE: hardcoded — make this a FRONTEND_URL env var once the server + // is deployed somewhere with a non-localhost frontend host. redirectURL := "http://localhost:3000/dashboard?token=" + token return c.Redirect(redirectURL, fiber.StatusFound) } diff --git a/autoship-server/internal/api/deploy.go b/autoship-server/internal/api/deploy.go new file mode 100644 index 0000000..2d137a6 --- /dev/null +++ b/autoship-server/internal/api/deploy.go @@ -0,0 +1,146 @@ +package api + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/cloud" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/db" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/models" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/services" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/utils" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// maxConcurrentDeploys caps how many builds run at the same time. Docker +// build + temp container probing is heavy; without a cap, a burst of requests +// can OOM the host. +const maxConcurrentDeploys = 8 + +// deploySlots is a counting semaphore — acquire by sending, release by +// receiving. Excess deploys block on send until a slot frees up. +var deploySlots = make(chan struct{}, maxConcurrentDeploys) + +type deployResult struct { + url string + projectType string + containerPort int + hostPort int + containerName string +} + +// runDeployment executes the build/deploy pipeline for a project that has +// already been persisted in the pending state. Runs in a detached goroutine +// from HandleRepoSubmit so HTTP-request cancellation does NOT abort the +// build mid-way (which would otherwise leave half-running containers). +// +// Status transitions: pending -> deploying -> (succeeded | failed). Failure +// detail lands in Project.DeployError so the client polling /projects/:id +// can surface it without needing log access. +func runDeployment(id primitive.ObjectID, username, repoURL, repoName, envContent, startCommand string) { + defer func() { + if r := recover(); r != nil { + log.Printf("[deploy %s] panic: %v", id.Hex(), r) + _ = db.UpdateProjectByID(id, bson.M{ + "status": models.StatusFailed, + "deploy_error": fmt.Sprintf("panic: %v", r), + }) + } + }() + + deploySlots <- struct{}{} + defer func() { <-deploySlots }() + + if err := db.UpdateProjectByID(id, bson.M{"status": models.StatusDeploying}); err != nil { + log.Printf("[deploy %s] failed to mark deploying: %v", id.Hex(), err) + // keep going — a transient Mongo blip shouldn't abort the build + } + + result, err := executeDeploy(username, repoURL, repoName, envContent, startCommand) + if err != nil { + log.Printf("[deploy %s] failed: %v", id.Hex(), err) + _ = db.UpdateProjectByID(id, bson.M{ + "status": models.StatusFailed, + "deploy_error": err.Error(), + }) + return + } + + if err := db.UpdateProjectByID(id, bson.M{ + "status": models.StatusSucceeded, + "project_type": result.projectType, + "hosted_url": result.url, + "container_port": result.containerPort, + "host_port": result.hostPort, + "container_name": result.containerName, + }); err != nil { + log.Printf("[deploy %s] succeeded but failed to persist result: %v", id.Hex(), err) + return + } + log.Printf("[deploy %s] succeeded (%s)", id.Hex(), result.url) +} + +func executeDeploy(username, repoURL, repoName, envContent, startCommand string) (deployResult, error) { + path, err := services.CloneRepository(repoURL, username, repoName) + if err != nil { + return deployResult{}, fmt.Errorf("clone failed: %w", err) + } + defer os.RemoveAll(path) + + projectType := services.DetectProjectType(path) + if projectType == "unknown" { + return deployResult{}, fmt.Errorf("unknown project type; no recognised entry file in repo") + } + + if projectType == "static" { + keyPrefix := fmt.Sprintf("%s/%s", username, repoName) + url, err := cloud.Get().UploadStaticSite(path, keyPrefix) + if err != nil { + return deployResult{}, fmt.Errorf("static upload: %w", err) + } + return deployResult{url: url, projectType: projectType}, nil + } + + // Dynamic: build + run container locally, then hand off to autoship-scripts + // (via the file-based queue) for DNS/nginx/SSL. + containerPort, hostPort, containerName, err := services.FullPipeline(username, path, envContent, startCommand) + if err != nil { + return deployResult{}, fmt.Errorf("container pipeline: %w", err) + } + + subdomain := utils.GenerateSubdomain(repoName, os.Getenv("DOMAIN")) + reqID := utils.GenerateRandomID() + deployReq := map[string]interface{}{ + "id": reqID, + "subdomain": subdomain, + "projectType": projectType, + "port": hostPort, + "status": "pending", + } + if err := utils.AppendJSONToFile("/var/lib/autoship/deploy/deploy-requests.json", deployReq); err != nil { + return deployResult{}, fmt.Errorf("queue deploy request: %w", err) + } + + resp, err := utils.WaitForResponse("/var/lib/autoship/deploy/deploy-responses.json", reqID, 60*time.Second) + if err != nil { + return deployResult{}, fmt.Errorf("wait for autoship-scripts: %w", err) + } + if status, _ := resp["status"].(string); status != "success" { + return deployResult{}, fmt.Errorf("autoship-scripts reported failure: %v", resp["error"]) + } + + url, _ := resp["url"].(string) + if url == "" { + url = fmt.Sprintf("https://%s", subdomain) + } + return deployResult{ + url: url, + projectType: projectType, + containerPort: containerPort, + hostPort: hostPort, + containerName: containerName, + }, nil +} diff --git a/autoship-server/internal/api/projects.go b/autoship-server/internal/api/projects.go index 4e2cce9..b44ed1d 100644 --- a/autoship-server/internal/api/projects.go +++ b/autoship-server/internal/api/projects.go @@ -2,162 +2,97 @@ package api import ( - "fmt" - "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/cloud" + "log" + "strings" + "time" + + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/config" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/db" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/models" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/services" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/utils" "github.com/gofiber/fiber/v2" "go.mongodb.org/mongo-driver/bson" - "log" - "os" - // "os/exec" - "strings" - "time" + "go.mongodb.org/mongo-driver/bson/primitive" ) -// RepoRequest struct defines the structure of the request for submitting a repo +// RepoRequest is the body shape of POST /projects/submit. type RepoRequest struct { RepoURL string `json:"repoURL"` - EnvContent string `json:"envContent,omitempty"` // Optional field for .env content + EnvContent string `json:"envContent,omitempty"` StartCommand string `json:"startCommand"` - // IN future, add fields for branch, commit - } -// HandleRepoSubmit handles the submission of a GitHub repository URL, -// clones the repository, detects the project type, and handles the hosting. +// HandleRepoSubmit accepts a deploy request, persists a pending Project, and +// kicks off the build in a detached goroutine. Returns 202 with the deployment +// id so the client can poll GET /projects/:id for status. Running the build +// inline would exceed typical browser/proxy timeouts on real repos (docker +// build + Azure NSG poll can run several minutes). func HandleRepoSubmit(c *fiber.Ctx) error { - domain := os.Getenv("DOMAIN") // e.g. a.com (optional) - // ec2IP := os.Getenv("EC2_PUBLIC_IP") var req RepoRequest if err := c.BodyParser(&req); err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid request") } if req.RepoURL == "" { - log.Println("repoURL not provided in request body") return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "repoURL is required"}) } - // fmt.Println("Received repo URL:", req.RepoURL) - - // Extract username from the GitHub repo URL username, err := utils.ExtractUsernameFromRepoURL(req.RepoURL) if err != nil { - log.Printf("Error extracting username: %v", err) return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": err.Error()}) } - - // Extract repo name from the URL parts := strings.Split(strings.TrimSuffix(req.RepoURL, ".git"), "/") repoName := parts[len(parts)-1] - // Clone the repository - path, err := services.CloneRepository(req.RepoURL, username, repoName) - if err != nil { - return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + now := time.Now() + project := &models.Project{ + ID: primitive.NewObjectID(), + Username: username, + RepoURL: req.RepoURL, + RepoName: repoName, + Status: models.StatusPending, + StartCommand: req.StartCommand, + CreatedAt: now, + UpdatedAt: now, } - - // Detect the type of the project (static or dynamic) - projectType := services.DetectProjectType(path) - if projectType == "unknown" { - // _ = os.RemoveAll(path) - return fiber.NewError(fiber.StatusBadRequest, "Unknown project type. Please ensure the repository contains a valid project structure.") + if err := db.SaveProject(project); err != nil { + return fiber.NewError(fiber.StatusInternalServerError, "Failed to record deployment") } - fmt.Println("Project type detected:", projectType) - var hostedURL string - var containerPort, hostPort int - var containerName string - // If the project is static, upload to S3/Blob and generate a hosted URL - if projectType == "static" { - keyPrefix := fmt.Sprintf("%s/%s", username, repoName) - url, err := cloud.Get().UploadStaticSite(path, keyPrefix) - if err != nil { - return fiber.NewError(fiber.StatusInternalServerError, "Failed to upload to S3: "+err.Error()) - } - _ = os.RemoveAll(path) - - //when returning the s3 url shorten it and encrypt it and in future all links we be routed through a proxy server - hostedURL = url - } else { - // Run FullPipeline to detect environment, write Dockerfile, build & run - containerPort, hostPort, containerName, err = services.FullPipeline(username, path, req.EnvContent, req.StartCommand) - // returns hostPort - subdomain := utils.GenerateSubdomain(repoName, domain) - // subdomain := fmt.Sprintf("%s.%s", repoName, domain)+ - if err != nil { - _ = os.RemoveAll(path) - return fiber.NewError(fiber.StatusInternalServerError, "Failed to deploy dynamic project: "+err.Error()) - } - - ec2Host := os.Getenv("EC2_PUBLIC_IP") // Set this in your .env or config - if ec2Host == "" { - ec2Host = "localhost" - } - - // hostedURL = fmt.Sprintf("http://%s:%d", ec2Host, hostPort) - hostedURL = fmt.Sprintf("https://%s", subdomain) - // Step 1: Create deployment request JSON - requestID := utils.GenerateRandomID() - deployRequest := map[string]interface{}{ - "id": requestID, - "subdomain": subdomain, - "projectType": projectType, - "port": hostPort, - "status": "pending", - } - - // Step 2: Append to /tmp/deploy-requests.json - if err := utils.AppendJSONToFile("/var/lib/autoship/deploy/deploy-requests.json", deployRequest); err != nil { - return fiber.NewError(fiber.StatusInternalServerError, "Failed to queue deployment: "+err.Error()) - } - - // Step 3: Wait for response (polling with timeout) - response, err := utils.WaitForResponse("/var/lib/autoship/deploy/deploy-responses.json", requestID, 60*time.Second) - if err != nil || response["status"] != "success" { - return fiber.NewError(fiber.StatusInternalServerError, fmt.Sprintf("Deployment failed: %v", response["error"])) - } - hostedURL = response["url"].(string) + + go runDeployment(project.ID, username, req.RepoURL, repoName, req.EnvContent, req.StartCommand) + + return c.Status(fiber.StatusAccepted).JSON(fiber.Map{ + "deploymentID": project.ID.Hex(), + "status": project.Status, + }) +} + +// GetProjectStatus returns the current state of a deployment by its id. The +// client polls this while Status is pending or deploying. Scoped to the +// authenticated user — ObjectIDs aren't unguessable enough to skip the check. +func GetProjectStatus(c *fiber.Ctx) error { + objID, err := primitive.ObjectIDFromHex(c.Params("id")) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, "invalid deployment id") } - // encrypt the hosted URL for security - // Create a new project model - project := &models.Project{ - Username: username, - RepoURL: req.RepoURL, - RepoName: repoName, - ProjectType: projectType, - HostedURL: hostedURL, - // add columns for createdAt, updatedAt, etc. - // ports will be added in future - // start command: "", - StartCommand: req.StartCommand, - ContainerPort: containerPort, - HostPort: hostPort, - ContainerName: containerName, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + project, err := db.GetProjectByID(objID) + if err != nil { + return fiber.NewError(fiber.StatusNotFound, "deployment not found") } - // Save the project details to the database - if err := db.SaveProject(project); err != nil { - return fiber.NewError(fiber.StatusInternalServerError, "Failed to save project") + claims := c.Locals("user").(*utils.Claims) + if project.Username != claims.Email { + return fiber.NewError(fiber.StatusNotFound, "deployment not found") } - // Return a success response with project details - return c.JSON(fiber.Map{ - "message": "Repository cloned and hosted successfully", - "projectType": projectType, - "url": hostedURL, - }) + return c.JSON(project) } -// GetUserProjects fetches all projects belonging to the authenticated user +// GetUserProjects fetches all projects belonging to the authenticated user. func GetUserProjects(c *fiber.Ctx) error { - // import your utils package if not already imported claims := c.Locals("user").(*utils.Claims) - username := claims.Email // or claims.UserID, depending on what you want + username := claims.Email collection := db.GetCollection("projects") cursor, err := collection.Find(c.Context(), bson.M{"username": username}) @@ -174,24 +109,26 @@ func GetUserProjects(c *fiber.Ctx) error { return c.JSON(projects) } -// DeleteDeployment deletes a project deployment by container name +// DeleteDeployment deletes a project deployment by container name. func DeleteDeployment(c *fiber.Ctx) error { containerName := c.Params("containerName") if containerName == "" { return fiber.NewError(fiber.StatusBadRequest, "containerName is required") } - // Use the new service to delete the project if err := services.DeleteProject(containerName); err != nil { log.Printf("Failed to delete project deployment for container %s: %v", containerName, err) return fiber.NewError(fiber.StatusInternalServerError, "Failed to delete deployment") } - // Remove from DB if err := db.DeleteProjectByContainerName(containerName); err != nil { log.Printf("Failed to delete project from DB for container %s: %v", containerName, err) - // Depending on desired behavior, you might want to return an error here - // For now, we log it and consider the primary operation (container removal) successful. + } + + // Free the host port for reuse. Non-fatal: the container is gone and the + // OS port is unbound regardless; the worst case is a stale "used" doc. + if err := db.ReleasePort(config.Get().MongoCollection, containerName); err != nil { + log.Printf("Failed to release port for container %s: %v", containerName, err) } return c.JSON(fiber.Map{"message": "Deployment deleted successfully"}) diff --git a/autoship-server/internal/api/router.go b/autoship-server/internal/api/router.go index 8eea13f..208e155 100644 --- a/autoship-server/internal/api/router.go +++ b/autoship-server/internal/api/router.go @@ -40,6 +40,7 @@ func registerAuthRoutes(app *fiber.App) { func registerProjectRoutes(app *fiber.App) { app.Post("/projects/submit", middleware.IsAuthenticated, HandleRepoSubmit) app.Get("/projects", middleware.IsAuthenticated, GetUserProjects) + app.Get("/projects/:id", middleware.IsAuthenticated, GetProjectStatus) app.Delete("/projects/:containerName", middleware.IsAuthenticated, DeleteDeployment) } diff --git a/autoship-server/internal/config/config.go b/autoship-server/internal/config/config.go new file mode 100644 index 0000000..ba2fb97 --- /dev/null +++ b/autoship-server/internal/config/config.go @@ -0,0 +1,76 @@ +// Package config centralizes environment-variable parsing so that .env is +// loaded exactly once at startup. Callers do config.Load() in main, then +// every other package reads typed fields via config.Get() instead of +// calling os.Getenv directly. +// +// Scope: this owns the env vars that previously had duplicate godotenv.Load +// sites (server wiring, Mongo, port allocator) plus what main needs to boot. +// Cloud-specific vars (S3_*, AZURE_*) and JWT secrets are still read lazily +// where they're used — those call sites run after Load, so they see the +// populated env without needing to own the load step. +package config + +import ( + "fmt" + "os" + "sync" + + "github.com/joho/godotenv" +) + +type Config struct { + ServerPort string + MongoURI string + MongoCollection string // collection name used by the port allocator + CloudProvider string +} + +var ( + cfg *Config + loadOnce sync.Once + loadErr error +) + +// Load reads .env (best-effort — a missing .env is fine for containerised +// runs where env vars come from the orchestrator) and resolves required +// environment variables into a typed Config. Idempotent: subsequent calls +// return the same instance without re-reading anything. +func Load() (*Config, error) { + loadOnce.Do(func() { + _ = godotenv.Load() + + mongoURI := os.Getenv("MONGO_URI") + if mongoURI == "" { + loadErr = fmt.Errorf("MONGO_URI is required") + return + } + + mongoCollection := os.Getenv("MONGO_DB_COLLECTION") + if mongoCollection == "" { + mongoCollection = "ports" + } + + serverPort := os.Getenv("PORT") + if serverPort == "" { + serverPort = "3000" + } + + cfg = &Config{ + ServerPort: serverPort, + MongoURI: mongoURI, + MongoCollection: mongoCollection, + CloudProvider: os.Getenv("CLOUD_PROVIDER"), + } + }) + return cfg, loadErr +} + +// Get returns the loaded config. Panics if Load wasn't called first — a +// programmer error caught at startup, not a runtime condition worth +// handling at every call site. +func Get() *Config { + if cfg == nil { + panic("config.Get() called before config.Load()") + } + return cfg +} diff --git a/autoship-server/internal/db/DeleteProjects.go b/autoship-server/internal/db/DeleteProjects.go index 59cb1e5..f15c637 100644 --- a/autoship-server/internal/db/DeleteProjects.go +++ b/autoship-server/internal/db/DeleteProjects.go @@ -10,6 +10,6 @@ func DeleteProjectByContainerName(containerName string) error { collection := GetCollection("projects") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err := collection.DeleteOne(ctx, bson.M{"containername": containerName}) + _, err := collection.DeleteOne(ctx, bson.M{"container_name": containerName}) return err } diff --git a/autoship-server/internal/db/indexes.go b/autoship-server/internal/db/indexes.go new file mode 100644 index 0000000..5d4d553 --- /dev/null +++ b/autoship-server/internal/db/indexes.go @@ -0,0 +1,32 @@ +package db + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// EnsurePortsIndex creates a unique index on the `port` field of the ports +// collection. The unique constraint is what makes GetOrReserveValidFreePort +// race-safe: concurrent InsertOne calls for the same port get a duplicate-key +// error rather than both succeeding (which was the silent bug in the prior +// upsert-with-$setOnInsert pattern). +// +// Idempotent: Mongo returns the existing index name if it's already present +// with the same key+options. Will FAIL if the collection currently holds +// duplicate port docs (a remnant of the old buggy upsert behavior); operators +// must dedupe before startup will succeed. +func EnsurePortsIndex(collectionName string) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + coll := GetCollection(collectionName) + _, err := coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.D{{Key: "port", Value: 1}}, + Options: options.Index().SetUnique(true).SetName("port_unique"), + }) + return err +} diff --git a/autoship-server/internal/db/ports.go b/autoship-server/internal/db/ports.go new file mode 100644 index 0000000..bd8daa7 --- /dev/null +++ b/autoship-server/internal/db/ports.go @@ -0,0 +1,40 @@ +package db + +import ( + "context" + "fmt" + "time" + + "go.mongodb.org/mongo-driver/bson" +) + +// ReleasePort marks the port owned by containerName as available, so that +// GetOrReserveValidFreePort's recycle branch can hand it to a future deploy +// instead of skipping past it forever as a watermarked phantom. +// +// Static projects don't have a port reservation; for them this is a no-op +// (MatchedCount == 0 is treated as success rather than an error, so callers +// can invoke this unconditionally on every delete). +// +// The bson key is "containerName" (camelCase) to match the PortMapping +// model's tag — DELIBERATELY different from Project's snake_case +// "container_name" tag. The two collections use inconsistent naming +// conventions; unifying them would require a data migration, out of scope +// here. +func ReleasePort(collectionName, containerName string) error { + coll := GetCollection(collectionName) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := coll.UpdateOne(ctx, + bson.M{"containerName": containerName}, + bson.M{"$set": bson.M{ + "status": "available", + "timestamp": time.Now(), + }}, + ) + if err != nil { + return fmt.Errorf("release port for %s: %w", containerName, err) + } + return nil +} diff --git a/autoship-server/internal/db/projects.go b/autoship-server/internal/db/projects.go index ba96d17..88d9648 100644 --- a/autoship-server/internal/db/projects.go +++ b/autoship-server/internal/db/projects.go @@ -5,11 +5,13 @@ import ( "time" "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/models" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" ) // SaveProject inserts a project document into the "projects" collection. func SaveProject(project *models.Project) error { - collection := GetCollection("projects") // use GetCollection from mongo.go + collection := GetCollection("projects") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -17,6 +19,27 @@ func SaveProject(project *models.Project) error { return err } -// func GetCollection(name string) *mongo.Collection { -// return Client.Database("autoship").Collection(name) -// } +// UpdateProjectByID applies the given $set fields to a project. updated_at +// is stamped automatically so callers don't have to remember. +func UpdateProjectByID(id primitive.ObjectID, fields bson.M) error { + collection := GetCollection("projects") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + fields["updated_at"] = time.Now() + _, err := collection.UpdateOne(ctx, bson.M{"_id": id}, bson.M{"$set": fields}) + return err +} + +// GetProjectByID returns a single project by its Mongo _id. +func GetProjectByID(id primitive.ObjectID) (*models.Project, error) { + collection := GetCollection("projects") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var p models.Project + if err := collection.FindOne(ctx, bson.M{"_id": id}).Decode(&p); err != nil { + return nil, err + } + return &p, nil +} diff --git a/autoship-server/internal/models/projects.go b/autoship-server/internal/models/projects.go index 061904b..dda4e60 100644 --- a/autoship-server/internal/models/projects.go +++ b/autoship-server/internal/models/projects.go @@ -1,8 +1,18 @@ package models import ( - "go.mongodb.org/mongo-driver/bson/primitive" "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// Deployment status values. A project moves pending -> deploying -> +// (succeeded | failed). Stored in Project.Status. +const ( + StatusPending = "pending" + StatusDeploying = "deploying" + StatusSucceeded = "succeeded" + StatusFailed = "failed" ) type Project struct { @@ -11,6 +21,8 @@ type Project struct { RepoURL string `bson:"repo_url" json:"repo_url"` RepoName string `bson:"repo_name" json:"repo_name"` ProjectType string `bson:"project_type" json:"project_type"` + Status string `bson:"status" json:"status"` + DeployError string `bson:"deploy_error,omitempty" json:"deploy_error,omitempty"` HostedURL string `bson:"hosted_url" json:"hosted_url"` StartCommand string `bson:"start_command" json:"start_command"` ContainerPort int `bson:"container_port" json:"container_port"` diff --git a/autoship-server/internal/services/clone.go b/autoship-server/internal/services/clone.go index e9e4aee..04b0877 100644 --- a/autoship-server/internal/services/clone.go +++ b/autoship-server/internal/services/clone.go @@ -13,6 +13,11 @@ import ( func CloneRepository(repoURL, username, repoName string) (string, error) { path := fmt.Sprintf("static/%s/%s", username, repoName) + // FUTURE: concurrent same-repo race — path has no deploy id, so two + // deploys of the same repo by the same user share this dir. The second + // sees it exists and reuses the first's in-flight clone; when the first + // finishes, its defer os.RemoveAll yanks the second's working tree + // mid-build. Thread a deployment id / timestamp suffix through. if _, err := os.Stat(path); err == nil { return path, nil // already cloned } diff --git a/autoship-server/internal/services/dynamic.go b/autoship-server/internal/services/dynamic.go index 3e8c283..bc34d68 100644 --- a/autoship-server/internal/services/dynamic.go +++ b/autoship-server/internal/services/dynamic.go @@ -90,6 +90,10 @@ func GenerateDockerfile(env Environment, repoPath, startCommand string) error { case EnvGo: baseImage = "golang:1.20" netToolsInstall = "RUN apt update && apt install -y net-tools" + // FUTURE: this builds `app` then CMDs whatever startCommand the user + // gave (often `go run main.go`), which doesn't match. Either set + // CMD ["./app"] for Go projects and ignore startCommand, or drop the + // build step and let `go run` handle it. installCmd = "RUN go build -o app ." default: return fmt.Errorf("unsupported environment: %s", env) @@ -223,6 +227,10 @@ func buildAndRunContainerHybrid(repoPath, containerName string) (int, int, error return 0, 0, fmt.Errorf("failed to find free host port: %w", err) } + // FUTURE: redundant — GetOrReserveValidFreePort already authorized this + // port on the cloud firewall internally. On Azure this adds ~10-30s + // for the NSG poll. Safe to delete once we're confident nothing else + // depends on AuthorizePort being called at this exact point. fmt.Println("Authorizing host port via cloud firewall: ", hostPort) if err := cloud.Get().AuthorizePort(hostPort); err != nil { _ = exec.Command("docker", "rm", "-f", tmpContainer).Run() @@ -243,6 +251,11 @@ func buildAndRunContainerHybrid(repoPath, containerName string) (int, int, error ) finalCmd.Stdout = os.Stdout finalCmd.Stderr = os.Stderr + // FUTURE: if docker bind fails here, hostPort was already reserved + // (ports doc + cloud firewall rule) but is never actually bound to a + // container — the reservation stays "used" forever. Wrap this function + // with a `bound bool` + defer db.ReleasePort so failures return the + // reservation to the recycle pool. if err := finalCmd.Run(); err != nil { return 0, 0, fmt.Errorf("docker final run failed: %w", err) } @@ -273,6 +286,9 @@ func FullPipeline(username, repoPath, envContent, startCommand string) (int, int // Step 4: Derive container name from repo repoName := filepath.Base(repoPath) // containerName := fmt.Sprintf("autoship-%s-%s", username, strings.ToLower(repoName)) + // FUTURE: Unix() is seconds resolution — two deploys of the same repo + // landing in the same wall-clock second collide on container name. + // Switch to UnixNano, or thread the deployment's Mongo _id through. timestamp := time.Now().Unix() containerName := fmt.Sprintf("autoship-%s-%s-%d", username, strings.ToLower(repoName), timestamp) diff --git a/autoship-server/internal/utils/jwt.go b/autoship-server/internal/utils/jwt.go index 35e8499..2131ac3 100644 --- a/autoship-server/internal/utils/jwt.go +++ b/autoship-server/internal/utils/jwt.go @@ -7,7 +7,6 @@ import ( "time" "github.com/golang-jwt/jwt/v4" - "github.com/joho/godotenv" ) var jwtKey []byte @@ -20,13 +19,10 @@ type Claims struct { jwt.RegisteredClaims } -// LoadEnv loads environment variables from the .env file +// LoadEnv reads JWT-related environment variables into package state. +// Assumes .env has already been loaded by config.Load — this function +// does NOT load .env itself (centralized loading lives in config). func LoadEnv() error { - if err := godotenv.Load(); err != nil { - return fmt.Errorf("error loading .env file: %w", err) - } - - // Set JWT secret key and expiration time from environment variables jwtKey = []byte(os.Getenv("JWT_SECRET")) expiration, err := time.ParseDuration(os.Getenv("JWT_EXPIRATION")) diff --git a/autoship-server/internal/utils/port_manager.go b/autoship-server/internal/utils/port_manager.go index 919e221..75f4c2e 100644 --- a/autoship-server/internal/utils/port_manager.go +++ b/autoship-server/internal/utils/port_manager.go @@ -3,133 +3,139 @@ package utils import ( "context" "fmt" - "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/cloud" - "github.com/joho/godotenv" "log" "net" - "os" "os/exec" "strconv" "strings" "time" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/cloud" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/config" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/db" + "github.com/Ashmit-Kumar/Auto-Ship/autoship-server/internal/models" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) -var ( - MongoURI string - DatabaseName string - CollectionName string -) - -func init() { - _ = godotenv.Load(".env") // Loads .env file from current directory (ignore error if not present) - - MongoURI = os.Getenv("MONGO_URI") - DatabaseName = os.Getenv("MONGO_DB_NAME") - CollectionName = os.Getenv("MONGO_DB_COLLECTION") -} - -// // GetOrReserveValidFreePort finds an unused port and opens it in the EC2 security group -// func GetOrReserveValidFreePort(containerName string) (int, error) { -// ctx := context.Background() - -// // 1. Connect to MongoDB -// client, err := mongo.Connect(ctx, options.Client().ApplyURI(MongoURI)) -// if err != nil { -// return 0, err -// } -// defer client.Disconnect(ctx) - -// coll := client.Database(DatabaseName).Collection(CollectionName) - -// // 2. Search for an available port in DB -// var portDoc struct { -// Port int `bson:"port"` -// } -// err = coll.FindOneAndUpdate(ctx, bson.M{"status": "available"}, bson.M{ -// "$set": bson.M{"status": "used", "containerName": containerName, "timestamp": time.Now()}, -// }).Decode(&portDoc) -// if err == mongo.ErrNoDocuments { -// // No free ports in DB; optionally generate a new one -// return 0, fmt.Errorf("no free ports in DB") -// } else if err != nil { -// return 0, err -// } - -// // 3. Check if it's free on this machine -// if !IsPortAvailable(portDoc.Port) { -// // Update status back to "available" -// _, _ = coll.UpdateOne(ctx, bson.M{"port": portDoc.Port}, bson.M{"$set": bson.M{"status": "available"}}) -// return GetOrReserveValidFreePort(containerName) -// } - -// // 4. Open port in EC2 Security Group -// if err := AuthorizeEC2Port(portDoc.Port); err != nil { -// log.Printf("Failed to open port %d in SG: %v", portDoc.Port, err) -// return 0, err -// } - -// return portDoc.Port, nil -// } - +// maxRecycleAttempts bounds how many "available" docs we'll try before +// giving up on the recycle pool and falling through to a fresh allocation. +// Protects against an unbounded loop if the firewall is broken and every +// reauth fails (we'd just keep marking docs available and re-picking them). +const maxRecycleAttempts = 5 + +// GetOrReserveValidFreePort allocates a host port for a new container. +// +// Two-stage strategy: +// +// 1. Recycle pool — try the lowest-numbered doc with status "available" +// (set by db.ReleasePort when a previous container was deleted). The +// cloud firewall rule from the previous owner is intentionally left +// alive; reauthorize is a no-op (AWS dedupes by description, Azure by +// rule name), so recycling skips the NSG/SG round-trip in the common +// case. +// 2. Fresh allocation — watermark scan above the highest port currently +// in the collection, insert a new doc atomically. Unique index on +// `port` (db.EnsurePortsIndex) makes concurrent claims for the same +// port fail with a duplicate-key error; the loser falls through. +// +// Mongo client: reuses the long-lived pooled client from db.GetCollection +// rather than opening a fresh connection per call. func GetOrReserveValidFreePort(containerName string) (int, error) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + coll := db.GetCollection(config.Get().MongoCollection) + + // Stage 1: try to recycle a freed port. + for attempt := 0; attempt < maxRecycleAttempts; attempt++ { + var recycled models.PortMapping + err := coll.FindOneAndUpdate( + ctx, + bson.M{"status": "available"}, + bson.M{"$set": bson.M{ + "status": "used", + "containerName": containerName, + "timestamp": time.Now(), + }}, + options.FindOneAndUpdate().SetSort(bson.D{{Key: "port", Value: 1}}), + ).Decode(&recycled) + + if err == mongo.ErrNoDocuments { + break // nothing recyclable; fall through to fresh allocation + } + if err != nil { + return 0, fmt.Errorf("failed to claim recyclable port: %w", err) + } - client, err := mongo.Connect(ctx, options.Client().ApplyURI(MongoURI)) - if err != nil { - return 0, err - } - defer client.Disconnect(ctx) + if !IsPortAvailable(recycled.Port) { + // Doc says available but the OS port is bound by something else. + // Prune the phantom and try the next available doc. + log.Printf("Recycled port %d not OS-free; pruning stale entry", recycled.Port) + _, _ = coll.DeleteOne(ctx, bson.M{"_id": recycled.ID}) + continue + } - coll := client.Database(DatabaseName).Collection(CollectionName) + if err := cloud.Get().AuthorizePort(recycled.Port); err != nil { + log.Printf("Failed to re-authorize recycled port %d: %v", recycled.Port, err) + // Put it back in the available pool — the cloud problem is likely + // transient and a later allocation should try again. + _, _ = coll.UpdateOne(ctx, + bson.M{"_id": recycled.ID}, + bson.M{"$set": bson.M{ + "status": "available", + "timestamp": time.Now(), + }}, + ) + continue + } + return recycled.Port, nil + } - // Step 1: Get the latest used or available port + // Stage 2: nothing recyclable worked. Scan above the watermark and insert. var portDoc struct { Port int `bson:"port"` } - opts := options.FindOne().SetSort(bson.D{{"port", -1}}) - err = coll.FindOne(ctx, bson.M{}, opts).Decode(&portDoc) - fmt.Println("Latest port found in DB:", portDoc.Port) + opts := options.FindOne().SetSort(bson.D{{Key: "port", Value: -1}}) + err := coll.FindOne(ctx, bson.M{}, opts).Decode(&portDoc) if err == mongo.ErrNoDocuments { - // No ports found, start from default - fmt.Println("No ports found in DB, starting from default port 2000") - portDoc.Port = 1999 // default fallback + log.Println("No ports found in DB, starting from 2000") + portDoc.Port = 1999 } else if err != nil { return 0, fmt.Errorf("failed to find latest port: %w", err) } - startPort := portDoc.Port + 1 // default fallback - // if err == nil { - // startPort = portDoc.Port+1 // start from the next port - // } + startPort := portDoc.Port + 1 - // Step 2: Try finding a free port by incrementing for port := startPort; port <= 65535; port++ { - if IsPortAvailable(port) { - // Try reserving in DB (ensure atomicity in multi-user environments) - _, err := coll.UpdateOne(ctx, - bson.M{"port": port, "status": bson.M{"$ne": "used"}}, // <--- important - bson.M{ - "$setOnInsert": bson.M{ - "port": port, - "status": "used", - "containerName": containerName, - "timestamp": time.Now(), - }, - }, - options.Update().SetUpsert(true), - ) - if err == nil { - // Open the port in the active cloud provider's firewall. - if err := cloud.Get().AuthorizePort(port); err != nil { - log.Printf("Failed to open port %d in firewall: %v", port, err) - continue - } - return port, nil + if !IsPortAvailable(port) { + continue + } + _, err := coll.InsertOne(ctx, models.PortMapping{ + Port: port, + ContainerPort: 0, + Status: "used", + ContainerName: containerName, + Timestamp: time.Now(), + }) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + continue } + return 0, fmt.Errorf("failed to reserve port %d: %w", port, err) } + if err := cloud.Get().AuthorizePort(port); err != nil { + log.Printf("Failed to open port %d in firewall: %v", port, err) + if _, delErr := coll.DeleteOne(ctx, bson.M{ + "port": port, + "containerName": containerName, + }); delErr != nil { + log.Printf("Failed to release Mongo reservation for port %d: %v", port, delErr) + } + continue + } + return port, nil } return 0, fmt.Errorf("no free ports found") @@ -144,7 +150,7 @@ func IsPortAvailable(port int) bool { return true } -// tryDefaultPorts checks common default ports like 3000, 5000, 8080 inside the container. +// tryDefaultPorts checks common default ports inside the container. func tryDefaultPorts(containerID string) (int, error) { defaultPorts := []int{3000, 5000, 8080, 80, 8000} for _, port := range defaultPorts { @@ -161,18 +167,14 @@ func tryDefaultPorts(containerID string) (int, error) { func detectPortWithNetstat(containerID string) (int, error) { fmt.Println("Running netstat inside the container to detect open ports... ", containerID) - // Sleep for few seconds to ensure the container is fully up - time.Sleep(5 * time.Second) // <-- wait for the container to be fully up + time.Sleep(5 * time.Second) // wait for the container to be fully up - // Execute netstat command inside the container cmd := exec.Command("docker", "exec", containerID, "netstat", "-tuln") if err := cmd.Run(); err != nil { return 0, fmt.Errorf("failed to exec netstat: %w", err) } - // output, err := cmd.Output() - // fmt.Println("Inspecting line:", line) - output, err := cmd.CombinedOutput() // <-- not just Output() - fmt.Println("Netstat Output:\n", string(output)) // <-- helpful to print it + output, err := cmd.CombinedOutput() + fmt.Println("Netstat Output:\n", string(output)) if err != nil { return 0, fmt.Errorf("failed to exec netstat: %w", err) } @@ -183,11 +185,10 @@ func detectPortWithNetstat(containerID string) (int, error) { if line == "" { continue } - fmt.Println("Inspecting line:", line) // <-- print each line for debugging - // Example line: "tcp 0 0 0.0.0.0:8080" + fmt.Println("Inspecting line:", line) fields := strings.Fields(line) if len(fields) >= 4 && (strings.HasPrefix(fields[0], "tcp") || strings.HasPrefix(fields[0], "udp")) { - addr := fields[3] // usually 0.0.0.0:8080 + addr := fields[3] if parts := strings.Split(addr, ":"); len(parts) > 1 { portStr := parts[len(parts)-1] if port, err := strconv.Atoi(portStr); err == nil { @@ -200,14 +201,11 @@ func detectPortWithNetstat(containerID string) (int, error) { } func DetectExposedPort(containerID string) (int, error) { - // Try default common ports first fmt.Println("Detecting Exposed Port using default ports... ", containerID) if port, err := tryDefaultPorts(containerID); err == nil { return port, nil } fmt.Println("No default port matched, trying dynamic detection...") - fmt.Println("Detecting port using netstat inside the container... ", containerID) - // Fallback to dynamic detection using netstat return detectPortWithNetstat(containerID) }