Add sitemap and sources preview (#180)

Generate sitemaps with functions

functions/[[accountId]]/widget/[[index]].js

@@ -1,4 +1,4 @@
-import { Buffer } from "node:buffer";
+import { socialGet, viewCall } from "../../common";
 
 class MetaTitleInjector {
   constructor({ title }) {
@@ -81,60 +81,6 @@ function defaultData() {
   };
 }
 
-async function socialGet(keys, blockHeight, parse) {
-  const request = await fetch("https://api.near.social/get", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      keys: [keys],
-      blockHeight,
-    }),
-  });
-  let data = await request.json();
-  const parts = keys.split("/");
-  for (let i = 0; i < parts.length; i++) {
-    const part = parts[i];
-    if (part === "*" || part === "**") {
-      break;
-    }
-    data = data?.[part];
-  }
-  if (parse) {
-    try {
-      data = JSON.parse(data);
-    } catch (e) {
-      return null;
-    }
-  }
-  return data;
-}
-
-async function viewCall({ contractId, method, args }) {
-  const res = await fetch("https://rpc.mainnet.near.org", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      jsonrpc: "2.0",
-      id: "dontcare",
-      method: "query",
-      params: {
-        request_type: "call_function",
-        finality: "final",
-        account_id: contractId,
-        method_name: method,
-        args_base64: btoa(JSON.stringify(args)),
-      },
-    }),
-  });
-  const json = await res.json();
-  const result = Buffer.from(json.result.result).toString("utf-8");
-  return JSON.parse(result);
-}
-
 async function nftToImageUrl({ contractId, tokenId }) {
   const [token, nftMetadata] = await Promise.all([
     viewCall({
@@ -282,6 +228,24 @@ async function widgetData(env, url, data) {
   data.accountId = accountId;
 }
 
+async function sourceData(env, url, data) {
+  const key = url.searchParams.get("src");
+  const parts = key.split("/");
+  const accountId = parts[0];
+  const blockHeight = url.searchParams.get("blockHeight");
+  const [source, image] = await Promise.all([
+    socialGet(key, blockHeight),
+    socialGet(`${key}/metadata/image/**`),
+  ]);
+
+  data.raw = source;
+  data.description = source;
+  data.image = null;
+  data.authorImage = await imageToUrl(env, image);
+  data.title = `Source code of ${key} at block height ${blockHeight} | Near Social`;
+  data.accountId = accountId;
+}
+
 async function generateData(env, url) {
   const data = defaultData();
   try {
@@ -291,6 +255,8 @@ async function generateData(env, url) {
       await postData(env, url, data, false);
     } else if (url.pathname === "/mob.near/widget/ProfilePage") {
       await profileData(env, url, data);
+    } else if (url.pathname === "/mob.near/widget/WidgetSource") {
+      await sourceData(env, url, data);
     } else {
       await widgetData(env, url, data);
     }

functions/common.js

@@ -0,0 +1,85 @@
+import { Buffer } from "node:buffer";
+
+export async function socialIndex(action, key, options) {
+  const request = await fetch("https://api.near.social/index", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      action,
+      key,
+      options,
+    }),
+  });
+  return await request.json();
+}
+
+export async function socialKeys(keys, blockHeight, options) {
+  const request = await fetch("https://api.near.social/keys", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      keys: [keys],
+      blockHeight,
+      options,
+    }),
+  });
+  return await request.json();
+}
+
+export async function socialGet(keys, blockHeight, parse) {
+  const request = await fetch("https://api.near.social/get", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      keys: [keys],
+      blockHeight,
+    }),
+  });
+  let data = await request.json();
+  const parts = keys.split("/");
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+    if (part === "*" || part === "**") {
+      break;
+    }
+    data = data?.[part];
+  }
+  if (parse) {
+    try {
+      data = JSON.parse(data);
+    } catch (e) {
+      return null;
+    }
+  }
+  return data;
+}
+
+export async function viewCall({ contractId, method, args }) {
+  const res = await fetch("https://rpc.mainnet.near.org", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      jsonrpc: "2.0",
+      id: "dontcare",
+      method: "query",
+      params: {
+        request_type: "call_function",
+        finality: "final",
+        account_id: contractId,
+        method_name: method,
+        args_base64: btoa(JSON.stringify(args)),
+      },
+    }),
+  });
+  const json = await res.json();
+  const result = Buffer.from(json.result.result).toString("utf-8");
+  return JSON.parse(result);
+}

functions/sitemap/index.js

@@ -0,0 +1,17 @@
+export async function onRequest({ request, next, env }) {
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <sitemap>
+        <loc>https://near.social/sitemap/posts/</loc>
+    </sitemap>
+    <sitemap>
+        <loc>https://near.social/sitemap/widgets/</loc>
+    </sitemap>
+    <sitemap>
+        <loc>https://near.social/sitemap/profiles/</loc>
+    </sitemap>
+    <sitemap>
+        <loc>https://near.social/sitemap/sources/</loc>
+    </sitemap>
+</sitemapindex>`);
+}

functions/sitemap/posts/[index].js

@@ -0,0 +1,33 @@
+import { socialIndex } from "../../common";
+
+const Limit = 50000;
+
+export const generateSitemapPosts = async (env, offset) => {
+  const posts = await socialIndex("post", "main", {
+    from: offset,
+    limit: Limit,
+  });
+  return posts
+    .map(
+      (post) =>
+        `  <url>
+    <loc>https://near.social/mob.near/widget/MainPage.Post.Page?accountId=${post.accountId}&blockHeight=${post.blockHeight}</loc>
+    <changefreq>never</changefreq>
+  </url>`
+    )
+    .join("\n");
+};
+
+export async function onRequest({ request, env, next }) {
+  const url = new URL(request.url);
+  const parts = url.pathname.split("/");
+  if (parts.length !== 4) {
+    return next();
+  }
+  const offset = parseInt(parts[3]);
+
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${await generateSitemapPosts(env, offset)}
+</urlset>`);
+}

functions/sitemap/posts/index.js

@@ -0,0 +1,8 @@
+export async function onRequest({}) {
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <sitemap>
+        <loc>https://near.social/sitemap/posts/0</loc>
+    </sitemap>
+</sitemapindex>`);
+}

functions/sitemap/profiles/index.js

@@ -0,0 +1,22 @@
+import { socialKeys } from "../../common";
+
+export const generateSitemapProfiles = async (env) => {
+  const data = await socialKeys("*/profile");
+  const accountIds = Object.keys(data);
+  return accountIds
+    .map(
+      (accountId) =>
+        `  <url>
+    <loc>https://near.social/mob.near/widget/ProfilePage?accountId=${accountId}</loc>
+    <changefreq>monthly</changefreq>
+  </url>`
+    )
+    .join("\n");
+};
+
+export async function onRequest({ env }) {
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${await generateSitemapProfiles(env)}
+</urlset>`);
+}

functions/sitemap/sources/index.js

@@ -0,0 +1,32 @@
+import { socialKeys } from "../../common";
+
+const MinBlockHeight = 75942518;
+
+export const generateSitemapSources = async (env) => {
+  const data = await socialKeys("*/widget/*", null, {
+    return_type: "History",
+  });
+  return Object.entries(data)
+    .map(([accountId, widget]) =>
+      Object.entries(widget.widget).map(([widgetId, blockHeights]) =>
+        blockHeights
+          .filter((blockHeight) => blockHeight >= MinBlockHeight)
+          .map(
+            (blockHeight) =>
+              `  <url>
+    <loc>https://near.social/mob.near/widget/WidgetSource?src=${accountId}/widget/${widgetId}&blockHeight=${blockHeight}</loc>
+    <changefreq>never</changefreq>
+  </url>`
+          )
+      )
+    )
+    .flat()
+    .join("\n");
+};
+
+export async function onRequest({ env }) {
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${await generateSitemapSources(env)}
+</urlset>`);
+}

functions/sitemap/widgets/index.js

@@ -0,0 +1,24 @@
+import { socialKeys } from "../../common";
+
+export const generateSitemapWidgets = async (env) => {
+  const data = await socialKeys("*/widget/*/metadata");
+  return Object.entries(data)
+    .map(([accountId, widget]) =>
+      Object.keys(widget.widget).map(
+        (widgetId) =>
+          `  <url>
+    <loc>https://near.social/${accountId}/widget/${widgetId}</loc>
+    <changefreq>monthly</changefreq>
+  </url>`
+      )
+    )
+    .flat()
+    .join("\n");
+};
+
+export async function onRequest({ env }) {
+  return new Response(`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${await generateSitemapWidgets(env)}
+</urlset>`);
+}

public/index.html

@@ -24,7 +24,7 @@
     <title>Near Social</title>
   </head>
   <body>
-    <noscript>
+    <noscript style="white-space: pre; font-family: monospace">
       You need to enable JavaScript to run this app.
     </noscript>
     <div id="root"></div>

public/robots.txt

@@ -1,3 +1,4 @@
 # https://www.robotstxt.org/robotstxt.html
+Sitemap: https://near.social/sitemap/
 User-agent: *
 Disallow: