Allow url documents to be injested, so it only hits the url once

app/jsonDoc.server.ts

@@ -1,4 +1,5 @@
 import { customRandom } from "nanoid";
+import safeFetch from "./utilities/safeFetch";
 
 type BaseJsonDocument = {
   id: string;
@@ -19,6 +20,7 @@ export type UrlJsonDocument = BaseJsonDocument & {
 export type CreateJsonOptions = {
   ttl?: number;
   readOnly?: boolean;
+  injest?: boolean;
   metadata?: any;
 };
 
@@ -42,7 +44,18 @@ export async function createFromUrl(
   title?: string,
   options?: CreateJsonOptions
 ): Promise<JSONDocument> {
+  if (options?.injest) {
+    const response = await safeFetch(url.href);
+
+    if (!response.ok) {
+      throw new Error(`Failed to injest ${url.href}`);
+    }
+
+    return createFromRawJson(title || url.href, await response.text(), options);
+  }
+
   const docId = createId();
+
   const doc: JSONDocument = {
     id: docId,
     type: <const>"url",

app/routes/new.tsx

@@ -14,6 +14,7 @@ export let loader: LoaderFunction = async ({ request, context }) => {
   const ttl = url.searchParams.get("ttl");
   const readOnly = url.searchParams.get("readonly");
   const title = url.searchParams.get("title");
+  const injest = url.searchParams.get("injest");
 
   if (!jsonUrl && !base64EncodedJson) {
     return redirect("/");
@@ -33,6 +34,10 @@ export let loader: LoaderFunction = async ({ request, context }) => {
     options.readOnly = readOnly === "true";
   }
 
+  if (typeof injest === "string") {
+    options.injest = injest === "true";
+  }
+
   if (jsonUrl) {
     const jsonURL = new URL(jsonUrl);