use self-hosted forest RPC instead of GLIF

.devcontainer/devcontainer.json

@@ -39,7 +39,9 @@
         "GOTMPDIR": "/home/vscode/.local/share/tmp",
         "PGDATA": "/home/vscode/.local/share/pg/pgdata",
         "PGPORT": "55432",
-        "PGSOCK_DIR": "/home/vscode/.local/share/pg"
+        "PGSOCK_DIR": "/home/vscode/.local/share/pg",
+        "FOREST_JWT": "${localEnv:FOREST_JWT}",
+        "SINGULARITY_TEST_EXTERNAL_API": "${localEnv:SINGULARITY_TEST_EXTERNAL_API}"
     },
     "runArgs": ["--userns=keep-id"],
     "containerUser": "vscode"

.github/workflows/devcontainer-podman.yml

@@ -54,13 +54,19 @@ jobs:
           disable-docker: true
           cache-storage: true
 
+      # containerEnv in devcontainer.json pulls these via ${localEnv:...} at
+      # container start; scope the secret to this step only. FOREST_JWT is
+      # empty on fork PRs (GitHub policy); SkipIfNotExternalAPI catches that.
       - name: Build devcontainer
         id: build
         uses: parkan/github-actions/devcontainer-build@v3
         with:
           workspace-folder: .
           container-runtime: podman
           container-id-label: ci=podman
+        env:
+          FOREST_JWT: ${{ secrets.FOREST_JWT }}
+          SINGULARITY_TEST_EXTERNAL_API: 'true'
 
       - name: Generate swagger code
         if: steps.codegen.outputs.needed == 'true'
@@ -103,7 +109,8 @@ jobs:
         with:
           container-id: ${{ steps.build.outputs.container-id }}
           command: >
-            cd /workspaces/singularity && mkdir -p artifacts && make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml"
+            cd /workspaces/singularity && mkdir -p artifacts &&
+            make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml"
           container-runtime: podman
 
       - name: Run integration tests

service/dealpusher/dealpusher_test.go

@@ -126,7 +126,7 @@ func (m *MockDealMaker) MakeDeal(ctx context.Context, actorObj model.Actor, car
 
 func TestDealMakerService_Start(t *testing.T) {
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		ctx, cancel := context.WithCancel(ctx)
 		exitErr := make(chan error, 1)
@@ -140,9 +140,9 @@ func TestDealMakerService_Start(t *testing.T) {
 
 func TestDealMakerService_MultipleInstances(t *testing.T) {
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service1, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service1, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
-		service2, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service2, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		ctx, cancel := context.WithTimeout(ctx, time.Second)
 		defer cancel()
@@ -161,7 +161,7 @@ func TestDealMakerService_FailtoSend(t *testing.T) {
 		waitPendingInterval = time.Minute
 	}()
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 2, 0)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 2, 0)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker
@@ -215,7 +215,7 @@ func TestDealMakerService_Cron(t *testing.T) {
 		waitPendingInterval = time.Minute
 	}()
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker
@@ -312,7 +312,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) {
 		waitPendingInterval = time.Minute
 	}()
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker
@@ -421,7 +421,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) {
 
 func TestDealmakerService_Force(t *testing.T) {
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker
@@ -480,7 +480,7 @@ func TestDealmakerService_Force(t *testing.T) {
 
 func TestDealMakerService_MaxReplica(t *testing.T) {
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 1)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 1)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker
@@ -537,7 +537,7 @@ func TestDealMakerService_MaxReplica(t *testing.T) {
 
 func TestDealMakerService_NewScheduleOneOff(t *testing.T) {
 	testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) {
-		service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10)
+		service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10)
 		require.NoError(t, err)
 		mockDealmaker := new(MockDealMaker)
 		service.dealMaker = mockDealmaker

service/dealtracker/ddo_tracking_test.go

@@ -202,7 +202,7 @@ func TestRunOnce_DoesNotEpochExpireDDODeals(t *testing.T) {
 		url, server := setupTestServerWithBody(t, `{}`)
 		defer server.Close()
 
-		tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, "", true)
+		tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, testutil.TestLotusToken, true)
 		require.NoError(t, tracker.runOnce(ctx))
 
 		var ddoActive model.Deal

service/dealtracker/dealtracker.go

@@ -442,7 +442,9 @@ func (d *DealTracker) runOnce(ctx context.Context) error {
 		return errors.Wrapf(err, "failed to get lotus head time from %s", d.lotusURL)
 	}
 
-	var lastEpoch int32
+	// derive from head so expiration works even if the market stream is empty;
+	// the trackDeal callback can still bump it higher if a deal reports newer
+	lastEpoch := int32(epochutil.TimeToEpoch(headTime))
 
 	db := d.dbNoContext.WithContext(ctx)
 	var actors []model.Actor

service/dealtracker/dealtracker_test.go

@@ -338,7 +338,7 @@ func TestRunOnce(t *testing.T) {
 		url, server := setupTestServerWithBody(t, string(body))
 		defer server.Close()
 		require.NoError(t, err)
-		tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true)
+		tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true)
 		err = tracker.runOnce(context.Background())
 		require.NoError(t, err)
 		var allDeals []model.Deal
@@ -403,7 +403,7 @@ func TestRunOnce_DoesNotEpochExpirePDPDeals(t *testing.T) {
 		url, server := setupTestServerWithBody(t, `{}`)
 		defer server.Close()
 
-		tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true)
+		tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true)
 		require.NoError(t, tracker.runOnce(ctx))
 
 		var marketDeal model.Deal

service/epochutil/epoch_test.go

@@ -9,36 +9,36 @@ import (
 )
 
 func TestDefaultValue(t *testing.T) {
-	err := Initialize(context.Background(), testutil.TestLotusAPI, "")
+	err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.EqualValues(t, 1598306400, GenesisTimestamp)
 }
 
 func TestCalibNet(t *testing.T) {
 	// This test may fail when calibnet resets
-	err := Initialize(context.Background(), "https://api.calibration.node.glif.io/rpc/v1", "")
+	err := Initialize(context.Background(), testutil.CalibnetRPC, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.EqualValues(t, 1667326380, GenesisTimestamp)
 }
 
 func TestEpochToTime(t *testing.T) {
-	err := Initialize(context.Background(), testutil.TestLotusAPI, "")
+	err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.EqualValues(t, 1598306400, GenesisTimestamp)
 	require.EqualValues(t, 1598306400, EpochToTime(0).Unix())
 	require.EqualValues(t, 1598306430, EpochToTime(1).Unix())
 }
 
 func TestUnixToEpoch(t *testing.T) {
-	err := Initialize(context.Background(), testutil.TestLotusAPI, "")
+	err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.EqualValues(t, 1598306400, GenesisTimestamp)
 	require.EqualValues(t, 0, UnixToEpoch(1598306400))
 	require.EqualValues(t, 1, UnixToEpoch(1598306430))
 }
 
 func TestTimeToEpoch(t *testing.T) {
-	err := Initialize(context.Background(), testutil.TestLotusAPI, "")
+	err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.EqualValues(t, 1598306400, GenesisTimestamp)
 	require.EqualValues(t, 0, TimeToEpoch(EpochToTime(0)))

util/testutil/anvil.go

@@ -41,11 +41,18 @@ func StartAnvil(t *testing.T, upstreamRPC string) *AnvilInstance {
 		t.Fatalf("finding free port: %v", err)
 	}
 
-	cmd := exec.Command("anvil",
+	args := []string{
 		"--fork-url", upstreamRPC,
 		"--port", fmt.Sprintf("%d", port),
 		"--silent",
-	)
+	}
+	// forest gateway needs bearer auth; anvil supports --fork-header
+	haveTok := os.Getenv("FOREST_JWT") != ""
+	if haveTok {
+		args = append(args, "--fork-header", "Authorization: Bearer "+os.Getenv("FOREST_JWT"))
+	}
+	t.Logf("anvil forking %s (auth=%t)", upstreamRPC, haveTok)
+	cmd := exec.Command("anvil", args...)
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	if err := cmd.Start(); err != nil {

util/testutil/calibnet.go

@@ -1,8 +1,11 @@
 package testutil
 
 // calibnet constants shared by integration tests
+
+// CalibnetRPC defaults to the self-hosted forest gateway; glif rate-limited CI.
+var CalibnetRPC = envOr("SINGULARITY_TEST_CALIBNET_RPC", "https://static.187.115.235.167.clients.your-server.de/calibnet/rpc/v1")
+
 const (
-	CalibnetRPC              = "https://api.calibration.node.glif.io/rpc/v1"
 	CalibnetChainID          = 314159
 	CalibnetDDOContract      = "0x889fD50196BE300D06dc4b8F0F17fdB0af587095"
 	CalibnetPaymentsContract = "0x09a0fDc2723fAd1A7b8e3e00eE5DF73841df55a0"

util/testutil/testutils.go

@@ -55,18 +55,37 @@ const TestWalletAddr = "f1fib3pv7jua2ockdugtz7viz3cyy6lkhh7rfx3sa"
 
 const TestPrivateKeyHex = "7b2254797065223a22736563703235366b31222c22507269766174654b6579223a226b35507976337148327349586343595a58594f5775453149326e32554539436861556b6c4e36695a5763453d227d"
 
-// TestLotusAPI is the Lotus API endpoint to use for tests
-// Using /rpc/v1 (stable) instead of deprecated /rpc/v0
-const TestLotusAPI = "https://api.node.glif.io/rpc/v1"
+// TestLotusAPI defaults to the self-hosted forest gateway; glif rate-limited CI.
+var TestLotusAPI = envOr("SINGULARITY_TEST_LOTUS_API", "https://static.187.115.235.167.clients.your-server.de/rpc/v1")
 
-// SkipIfNotExternalAPI skips the test if SINGULARITY_TEST_EXTERNAL_API is not set
-// Use this for tests that make external API calls (e.g., Lotus/Filecoin APIs)
-// These are skipped by default because external APIs may be unreliable or rate-limited
+// TestLotusToken is the bearer token for the forest gateway; empty in local
+// dev and on fork PRs (see SkipIfNotExternalAPI).
+var TestLotusToken = os.Getenv("FOREST_JWT")
+
+func envOr(name, def string) string {
+	if v := os.Getenv(name); v != "" {
+		return v
+	}
+	return def
+}
+
+// SkipIfNotExternalAPI skips the test if external API access isn't configured.
+// Two conditions must hold: SINGULARITY_TEST_EXTERNAL_API opts in, and either
+// FOREST_JWT is set or one of the endpoint overrides
+// (SINGULARITY_TEST_LOTUS_API / SINGULARITY_TEST_CALIBNET_RPC) points at
+// something that doesn't need the gateway's bearer auth. Fork PRs and local
+// devs without a token skip cleanly instead of hitting a 401.
 func SkipIfNotExternalAPI(t *testing.T) {
 	t.Helper()
 	if os.Getenv("SINGULARITY_TEST_EXTERNAL_API") == "" {
 		t.Skip("Skipping external API test. Set SINGULARITY_TEST_EXTERNAL_API=true to run")
 	}
+	if os.Getenv("FOREST_JWT") == "" &&
+		os.Getenv("SINGULARITY_TEST_LOTUS_API") == "" &&
+		os.Getenv("SINGULARITY_TEST_CALIBNET_RPC") == "" {
+		t.Skip("Skipping: FOREST_JWT not set and no endpoint override")
+	}
+	t.Logf("external API: lotus=%s calibnet=%s token=%t", TestLotusAPI, CalibnetRPC, TestLotusToken != "")
 }
 
 func EscapePath(p string) string {

util/util_test.go

@@ -25,24 +25,15 @@ func TestNextPowerOfTwo(t *testing.T) {
 
 func TestNewLotusClient(t *testing.T) {
 	testutil.SkipIfNotExternalAPI(t)
-	for _, token := range []string{""} {
-		t.Run(token, func(t *testing.T) {
-			client := NewLotusClient(testutil.TestLotusAPI, token)
-			resp, err := client.Call(context.Background(), "Filecoin.Version")
-			if token != "" {
-				require.Error(t, err)
-				require.ErrorContains(t, err, "401")
-				return
-			}
-			require.NoError(t, err)
-			require.NotNil(t, resp.Result)
-		})
-	}
+	client := NewLotusClient(testutil.TestLotusAPI, testutil.TestLotusToken)
+	resp, err := client.Call(context.Background(), "Filecoin.Version")
+	require.NoError(t, err)
+	require.NotNil(t, resp.Result)
 }
 
 func TestGetLotusHeadTime(t *testing.T) {
 	testutil.SkipIfNotExternalAPI(t)
-	headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, "")
+	headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken)
 	require.NoError(t, err)
 	require.NotZero(t, headTime)
 }