Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions packages/cli/src/capture/contentExtractor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,83 @@ describe("captionImagesWithGemini — Vertex AI provider", () => {
expect(clientOptions[0]).toMatchObject({ location: "europe-west4" });
});

it.each([
"attacker.example/",
"x.attacker.example#",
"a@attacker.example",
"attacker.example:443?",
"us-central1.attacker.example",
])("skips captioning for a location that is not a region: %s", async (location) => {
const dir = makeProjectWithImages();
dirs.push(dir);
vertexEnv();
vi.stubEnv("HYPERFRAMES_VERTEX_LOCATION", location);

const warnings: string[] = [];
const captions = await captionImagesWithGemini(dir, () => {}, warnings);

expect(captions).toEqual({});
expect(clientOptions).toHaveLength(0);
expect(generateContentMock).not.toHaveBeenCalled();
expect(warnings).toEqual([
"HYPERFRAMES_VERTEX_LOCATION is not a Google Cloud region; skipped vision captioning.",
]);
});

it.each([
"../../v1/other",
"gemini?alt=x",
"gemini#x",
"https://attacker.example/m",
"/models/x",
])("skips captioning for a model that is not a model id: %s", async (model) => {
const dir = makeProjectWithImages();
dirs.push(dir);
vertexEnv();
vi.stubEnv("HYPERFRAMES_VERTEX_MODEL", model);

const warnings: string[] = [];
const captions = await captionImagesWithGemini(dir, () => {}, warnings);

expect(captions).toEqual({});
expect(clientOptions).toHaveLength(0);
expect(warnings).toEqual([
"HYPERFRAMES_VERTEX_MODEL is not a model id; skipped vision captioning.",
]);
});

it.each([
"gemini-2.5-flash",
"google/gemini-2.5-flash",
"publishers/anthropic/models/claude-sonnet-4@20250514",
"projects/example.com:proj/locations/us-central1/endpoints/123",
])("accepts the model id %s", async (model) => {
const dir = makeProjectWithImages();
dirs.push(dir);
vertexEnv();
vi.stubEnv("HYPERFRAMES_VERTEX_MODEL", model);
generateContentMock.mockResolvedValue({ text: "A caption." });

const captions = await captionImagesWithGemini(dir, () => {}, []);

expect(captions).toEqual({ "hero.png": "A caption." });
expect(generateContentMock.mock.calls[0]?.[0]).toMatchObject({ model });
});

it.each(["global", "us", "eu", "europe-west4"])("accepts the region %s", async (location) => {
const dir = makeProjectWithImages();
dirs.push(dir);
vertexEnv();
vi.stubEnv("HYPERFRAMES_VERTEX_LOCATION", location);
vi.stubEnv("HYPERFRAMES_VERTEX_MODEL", "publishers/google/models/gemini-2.5-flash-001");
generateContentMock.mockResolvedValue({ text: "A caption." });

const captions = await captionImagesWithGemini(dir, () => {}, []);

expect(captions).toEqual({ "hero.png": "A caption." });
expect(clientOptions[0]).toMatchObject({ location });
});

it("spends no output budget on thinking", async () => {
// Thinking tokens come out of maxOutputTokens, so a model left free to think can consume
// the whole budget and return empty text: a successful request that produces no caption.
Expand Down
31 changes: 30 additions & 1 deletion packages/cli/src/capture/contentExtractor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,27 @@ function vertexCredentials(raw: string): Record<string, unknown> | string {
: "HYPERFRAMES_VERTEX_SERVICE_ACCOUNT is not a service-account key; skipped vision captioning.";
}

// The SDK builds the request host from the location and the request path from the model id.
// `:` stays legal inside a segment for endpoint paths in domain-scoped projects (`example.com:proj`).
const VERTEX_LOCATION = /^[a-z][a-z0-9-]*$/;
const GOOGLE_MODEL_ID = /^[A-Za-z0-9][\w.:@-]*(?:\/[A-Za-z0-9][\w.:@-]*)*$/;

function googleClientSettingsProblem(
provider: "vertex" | "gemini",
model: string,
location: string,
): string | undefined {
if (provider === "vertex" && !VERTEX_LOCATION.test(location)) {
return "HYPERFRAMES_VERTEX_LOCATION is not a Google Cloud region; skipped vision captioning.";
}
if (!GOOGLE_MODEL_ID.test(model)) {
const variable =
provider === "vertex" ? "HYPERFRAMES_VERTEX_MODEL" : "HYPERFRAMES_GEMINI_MODEL";
return `${variable} is not a model id; skipped vision captioning.`;
}
return undefined;
}

function resolveVisionRequestTimeoutMs(): number {
const configured = Number(process.env.HYPERFRAMES_VISION_TIMEOUT_MS);
return Number.isFinite(configured) && configured > 0
Expand Down Expand Up @@ -398,6 +419,14 @@ export async function captionImagesWithGemini(
}, timeoutMs);
};
} else {
const vertexLocation = process.env.HYPERFRAMES_VERTEX_LOCATION || "us-central1";
const settingsProblem = googleClientSettingsProblem(provider, model, vertexLocation);
if (settingsProblem) {
warnings.push(settingsProblem);
internalError = true;
reportOutcome();
return geminiCaptions;
}
let GoogleGenAI: OptionalPackageModules["@google/genai"]["GoogleGenAI"];
try {
({ GoogleGenAI } = await loadOptionalPackage("@google/genai", "vision captioning"));
Expand All @@ -421,7 +450,7 @@ export async function captionImagesWithGemini(
ai = new GoogleGenAI({
vertexai: true,
project: vertexProject,
location: process.env.HYPERFRAMES_VERTEX_LOCATION || "us-central1",
location: vertexLocation,
googleAuthOptions: { credentials },
});
} else {
Expand Down
Loading