feat(beta,docs): confine the egress prober in both deployment styles

.gitignore

@@ -3,6 +3,8 @@ beta-vault/config/**/*.mmdb
 beta-vault/vault/tls/**/*.pem
 beta-vault/vault/tls/**/*.pub.pem
 beta-vault/beta-secrets.env
+# egress-prober client jwt and operator secret; the .example template is tracked
+beta-vault/prober.env
 beta-vault/vault/*.yml
 # Keep .example templates tracked
 !beta-vault/vault/*.yml.example

Caddyfile

@@ -1,7 +1,38 @@
+# connect derives every client's location from the address it sees on the
+# websocket. It only trusts a forwarded address when BOTH X-Forwarded-For and
+# X-Forwarded-Source-Port are present (connect/transport.go:220-227) -- that
+# pair is what the warp load balancer sends. Caddy sets X-Forwarded-For on its
+# own, but not the port, so connect fell through to r.RemoteAddr and saw
+# Caddy's container address (172.18.x.x) for every client. A private address
+# has no mmdb entry, so GuessLocationType returned "Unknown location type",
+# no network_client_location row was written, and no provider ever reached
+# network_client_location_reliability -- which silently emptied the provider
+# location list.
+#
+# Sending the source port restores the pair, and with it the real client
+# address. api needs it too: the same address feeds rate limiting and the
+# ARIN foreign-provider score, both of which would otherwise see one shared
+# proxy address for the entire deployment.
+(client_address) {
+	# connect trusts X-UR-Forwarded-For ahead of everything else
+	# (connect/transport.go:220) and the api derives the session client address
+	# from the same value. A proxy passes client headers through verbatim, so
+	# with Caddy as the sole ingress any client could send this header and pin
+	# itself to an address of its choosing -- forging its provider location and
+	# evading the per-address rate limits. Strip it on the way in; only the
+	# warp load balancer is entitled to set it, and it is not in this path.
+	header_up -X-UR-Forwarded-For
+	header_up X-Forwarded-Source-Port {http.request.remote.port}
+}
+
 api.beta-test.net {
-	reverse_proxy api:8080
+	reverse_proxy api:8080 {
+		import client_address
+	}
 }
 
 connect.beta-test.net {
-	reverse_proxy connect:80
+	reverse_proxy connect:80 {
+		import client_address
+	}
 }

beta-setup.sh

@@ -157,11 +157,12 @@ docker compose -f docker-compose.beta.yml up -d api connect taskworker
 
 echo
 echo "Beta network running:"
-echo "  API:     http://$PUBLIC_IP:8080"
-echo "  Connect: ws://$PUBLIC_IP:5080/"
+echo "  API:     https://api.beta-test.net"
+echo "  Connect: wss://connect.beta-test.net/"
 echo
 echo "All generated secrets live in $SECRETS_FILE and beta-vault/vault/*.yml"
 echo "These files are NOT tracked by git."
 echo "Postgres data is preserved across ./beta-setup.sh runs in the named volume."
 echo "To intentionally wipe the database, run ./beta-down.sh -v."
-echo "To use from another machine, open TCP ports 8080, 5080, and 15080."
+echo "Caddy is the only ingress; open TCP 80 and 443. The api and connect"
+echo "service ports are deliberately unpublished -- see BETA.md, Firewall."

beta-vault/prober.env.example

@@ -0,0 +1,21 @@
+# Secrets for the egress-prober service in docker-compose.beta.yml.
+#
+# Copy to beta-vault/prober.env and fill both values in. The real file is
+# gitignored; this template is not. Never commit the real one.
+#
+#   cp beta-vault/prober.env.example beta-vault/prober.env
+#
+# UR_PROBER_BY_JWT
+#   A network CLIENT jwt for the network the prober probes from. Create a
+#   network, then POST /network/auth-client with the dashboard jwt and use the
+#   by_client_jwt it returns. It must contain a client_id -- a plain user jwt
+#   has none and the prober exits with "parse by-jwt client id".
+#
+# UR_OPERATOR_SECRET
+#   Must equal ingest_secret in beta-vault/vault/provider_egress.yml, which
+#   ./beta-setup.sh generates. Read it with:
+#     grep ingest_secret beta-vault/vault/provider_egress.yml
+#   The api process memoizes it on first use, so restart api after rotating it.
+
+UR_PROBER_BY_JWT=
+UR_OPERATOR_SECRET=

controller/network_client_controller.go

@@ -81,9 +81,17 @@ func SetConnectionLocation(
 		if egress.Proxy {
 			scores.NetTypePrivacy = 1
 		}
-		if egress.Mobile {
-			scores.NetTypeVirtual = 1
-		}
+		// egress.Mobile deliberately does NOT feed NetTypeVirtual: unlike
+		// Hosting/Proxy, Mobile has no mmdb-path equivalent (IpInfo has no
+		// Mobile concept; NetTypeVirtual is set from the ipinfo schema's
+		// is_satellite field only, see GetLocationForIp, and never from
+		// DB-IP). Deriving NetTypeVirtual from Mobile here would penalize a
+		// probed mobile provider's ranking with no equivalent penalty for an
+		// otherwise-identical unprobed one -- the opposite of the parity
+		// this feature is meant to preserve (see arinForeignScore's doc for
+		// the same parity reasoning applied to net_type_foreign). Mobile
+		// stays on the model/wire contract as metadata; it just does not
+		// feed the ranking score.
 		// keep the ARIN org-vs-country foreign check on the probed path too,
 		// so a probed provider is ranked on equal terms with an equivalent
 		// unprobed one (net_type_foreign feeds the ranking columns). Compute

controller/network_client_controller_test.go

@@ -274,7 +274,14 @@ func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T
 }
 
 // A fresh probed location's Hosting/Proxy flags must map onto the stored
-// connection's net_type_hosting/net_type_privacy scores.
+// connection's net_type_hosting/net_type_privacy scores. Mobile must NOT map
+// onto net_type_virtual: Hosting/Proxy have direct mmdb-path equivalents
+// (ipInfo.Hosting/ipInfo.Privacy, see GetLocationForIp), but Mobile has none
+// (IpInfo has no Mobile concept at all, and NetTypeVirtual is only ever set
+// from the ipinfo schema's is_satellite field, never from DB-IP or from
+// anything Mobile-shaped) -- deriving NetTypeVirtual from Mobile would give
+// a probed mobile provider a ranking penalty an identical unprobed mobile
+// provider never takes, breaking the parity this feature promises.
 func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) {
 	server.DefaultTestEnv().Run(t, func(t testing.TB) {
 		ctx := context.Background()
@@ -300,6 +307,7 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) {
 			CountryCode: "jp",
 			Hosting:     true,
 			Proxy:       true,
+			Mobile:      true,
 			ObservedAt:  server.NowUtc(),
 		})
 
@@ -308,19 +316,21 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) {
 
 		var netTypeHosting int
 		var netTypePrivacy int
+		var netTypeVirtual int
 		server.Db(ctx, func(conn server.PgConn) {
 			result, qerr := conn.Query(
 				ctx,
-				`SELECT net_type_hosting, net_type_privacy FROM network_client_location WHERE connection_id = $1`,
+				`SELECT net_type_hosting, net_type_privacy, net_type_virtual FROM network_client_location WHERE connection_id = $1`,
 				connectionId,
 			)
 			server.WithPgResult(result, qerr, func() {
 				if result.Next() {
-					server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy))
+					server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy, &netTypeVirtual))
 				}
 			})
 		})
 		assert.Equal(t, netTypeHosting, 1)
 		assert.Equal(t, netTypePrivacy, 1)
+		assert.Equal(t, netTypeVirtual, 0)
 	})
 }

controller/provider_egress_location_controller.go

@@ -14,6 +14,19 @@ import (
 // already older than this when it arrives. It bounds replay of an old probe.
 const MaxProviderEgressLocationSubmissionAge = 24 * time.Hour
 
+// MaxProviderEgressLocationSubmissionSkew rejects a submission whose
+// observed_at is further in the future than this. The prober and server
+// clocks should be roughly in sync, so a few minutes of allowance covers
+// ordinary clock drift without opening the door to a far-future timestamp.
+// Without this bound, a future observed_at would defeat every other
+// safeguard at once: it always wins the monotonic upsert in
+// model.SetProviderEgressLocation (so no later, legitimate probe can ever
+// overwrite it), it reads as "fresh" forever against
+// ProviderEgressLocationMaxAge, and it outlives the taskworker sweep in
+// RemoveExpiredProviderEgressLocations -- permanently pinning a provider to
+// whatever location was submitted, with no API-side recovery.
+const MaxProviderEgressLocationSubmissionSkew = 5 * time.Minute
+
 // maxLocationNameLen bounds country/city/region as submitted: these flow into
 // model.CreateLocation, whose location_name column is varchar(128). Rejecting
 // an over-long value here with a clear error is preferable to letting
@@ -66,6 +79,9 @@ func SubmitProviderEgressLocation(
 	if args.ObservedAt.Before(server.NowUtc().Add(-MaxProviderEgressLocationSubmissionAge)) {
 		return nil, fmt.Errorf("Submission is too old.")
 	}
+	if server.NowUtc().Add(MaxProviderEgressLocationSubmissionSkew).Before(args.ObservedAt) {
+		return nil, fmt.Errorf("Submission is too far in the future.")
+	}
 	if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil {
 		return nil, fmt.Errorf("Unknown client.")
 	}

controller/provider_egress_location_controller_test.go

@@ -293,3 +293,62 @@ func TestSubmitProviderEgressLocationRejectsStaleObservedAt(t *testing.T) {
 		}
 	})
 }
+
+// A far-future observed_at must be rejected, not just an old one: unchecked,
+// it would defeat the monotonic upsert (it always "wins"), read as fresh
+// forever, and outlive the taskworker sweep -- permanently pinning a
+// provider's location with no API-side recovery. See
+// MaxProviderEgressLocationSubmissionSkew.
+func TestSubmitProviderEgressLocationRejectsFutureObservedAt(t *testing.T) {
+	server.DefaultTestEnv().Run(t, func(t testing.TB) {
+		ctx := context.Background()
+
+		networkId := server.NewId()
+		clientId := server.NewId()
+		model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "")
+
+		_, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{
+			ClientId:         clientId,
+			CountryCode:      "jp",
+			Country:          "Japan",
+			CountryConfident: true,
+			ObservedAt:       server.NowUtc().Add(10 * 365 * 24 * time.Hour),
+		})
+		if err == nil {
+			t.Fatal("a far-future observed_at must be rejected")
+		}
+		if model.GetProviderEgressLocation(ctx, clientId) != nil {
+			t.Fatal("rejected submission must not be stored")
+		}
+	})
+}
+
+// A submission within the allowed clock-skew window must still be accepted:
+// the future-timestamp rejection must not be so strict that ordinary clock
+// drift between the prober and server breaks legitimate submissions.
+func TestSubmitProviderEgressLocationAcceptsWithinSkewObservedAt(t *testing.T) {
+	server.DefaultTestEnv().Run(t, func(t testing.TB) {
+		ctx := context.Background()
+
+		networkId := server.NewId()
+		clientId := server.NewId()
+		model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "")
+
+		res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{
+			ClientId:         clientId,
+			CountryCode:      "us",
+			Country:          "United States",
+			CountryConfident: true,
+			ObservedAt:       server.NowUtc().Add(1 * time.Minute),
+		})
+		assert.Equal(t, err, nil)
+		if res.LocationId == (server.Id{}) {
+			t.Fatal("expected a resolved location id")
+		}
+
+		stored := model.GetProviderEgressLocation(ctx, clientId)
+		if stored == nil {
+			t.Fatal("expected the submission to be stored")
+		}
+	})
+}