Add initial delay when loading python functions (#8239)

* Improve robustness of function discovery for python

Anecdotally, python function discovery is flakey. We propose 2 change in
this PR:

1. For python discovery, add a small initial delay for python's admin
server to boot.

2. Add a request timeout to retry call to retrieve trigger information.
Previously, the default timeout would've been set to OS-level TCP
timeout, which in my laptop was between 20~30s.

* Add changelog.

* Remove per-req timeout to accomodate loading large/slow main.py.

* Update changelog.

* Revert timeout bump.

Author: taeold

CHANGELOG.md

@@ -1 +1,2 @@
+- Add initial delay when loading python functions (#8239)
 - Enforce webframeworks enablement only on webframeworks sites (#8168)

src/deploy/functions/runtimes/discovery/index.spec.ts

@@ -101,9 +101,22 @@ describe("detectFromPort", () => {
       code: "ECONNREFUSED",
     });
 
+    nock("http://127.0.0.1:8080").get("/__/functions.yaml").times(3).replyWithError({
+      message: "Almost there",
+      code: "ETIMEDOUT",
+    });
+
     nock("http://127.0.0.1:8080").get("/__/functions.yaml").reply(200, YAML_TEXT);
 
     const parsed = await discovery.detectFromPort(8080, "project", "nodejs16");
     expect(parsed).to.deep.equal(BUILD);
   });
+
+  it("retries when request times out", async () => {
+    nock("http://127.0.0.1:8081").get("/__/functions.yaml").delay(1_000).reply(200, YAML_TEXT);
+    nock("http://127.0.0.1:8080").get("/__/functions.yaml").reply(200, YAML_TEXT);
+
+    const parsed = await discovery.detectFromPort(8080, "project", "nodejs16", 0, 500);
+    expect(parsed).to.deep.equal(BUILD);
+  });
 });

src/deploy/functions/runtimes/discovery/index.ts

@@ -75,6 +75,7 @@ export async function detectFromPort(
   port: number,
   project: string,
   runtime: Runtime,
+  initialDelay = 0,
   timeout = 10_000 /* 10s to boot up */,
 ): Promise<build.Build> {
   let res: Response;
@@ -84,13 +85,21 @@ export async function detectFromPort(
     }, getFunctionDiscoveryTimeout() || timeout);
   });
 
+  // Initial delay to wait for admin server to boot.
+  if (initialDelay > 0) {
+    await new Promise((resolve) => setTimeout(resolve, initialDelay));
+  }
+
+  const url = `http://127.0.0.1:${port}/__/functions.yaml`;
   while (true) {
     try {
-      res = await Promise.race([fetch(`http://127.0.0.1:${port}/__/functions.yaml`), timedOut]);
+      res = await Promise.race([fetch(url), timedOut]);
       break;
     } catch (err: any) {
-      // Allow us to wait until the server is listening.
-      if (err?.code === "ECONNREFUSED") {
+      if (
+        err?.name === "FetchError" ||
+        ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT"].includes(err?.code)
+      ) {
         continue;
       }
       throw err;

src/deploy/functions/runtimes/python/index.ts

@@ -195,7 +195,12 @@ export class Delegate implements runtimes.RuntimeDelegate {
       });
       const killProcess = await this.serveAdmin(adminPort, envs);
       try {
-        discovered = await discovery.detectFromPort(adminPort, this.projectId, this.runtime);
+        discovered = await discovery.detectFromPort(
+          adminPort,
+          this.projectId,
+          this.runtime,
+          500 /* initialDelay, python startup is slow */,
+        );
       } finally {
         await killProcess();
       }