diff --git a/client/web/src/enterprise/batches/settings/CommitSigningIntegrations.tsx b/client/web/src/enterprise/batches/settings/CommitSigningIntegrations.tsx
index 3cf376e7b5c..1b76ade0be0 100644
--- a/client/web/src/enterprise/batches/settings/CommitSigningIntegrations.tsx
+++ b/client/web/src/enterprise/batches/settings/CommitSigningIntegrations.tsx
@@ -16,11 +16,12 @@ import {
     SummaryContainer,
 } from '../../../components/FilteredConnection/ui'
 import { GitHubAppFailureAlert } from '../../../components/gitHubApps/GitHubAppFailureAlert'
-import type {
-    BatchChangesCodeHostFields,
-    GlobalBatchChangesCodeHostsResult,
-    Scalars,
-    UserBatchChangesCodeHostsResult,
+import {
+    type BatchChangesCodeHostFields,
+    GitHubAppKind,
+    type GlobalBatchChangesCodeHostsResult,
+    type Scalars,
+    type UserBatchChangesCodeHostsResult,
 } from '../../../graphql-operations'
 
 import { useGlobalBatchChangesCodeHostConnection, useUserBatchChangesCodeHostConnection } from './backend'
@@ -54,9 +55,12 @@ export const CommitSigningIntegrations: React.FunctionComponent<
     const { loading, hasNextPage, fetchMore, connection, error, refetchAll } = connectionResult
 
     const location = useLocation()
-    const success = new URLSearchParams(location.search).get('success') === 'true'
-    const appName = new URLSearchParams(location.search).get('app_name')
-    const setupError = new URLSearchParams(location.search).get('error')
+    const searchParams = new URLSearchParams(location.search)
+    const kind = searchParams.get('kind')
+    const success = searchParams.get('success') === 'true'
+    const appName = searchParams.get('app_name')
+    const setupError = searchParams.get('error')
+    const shouldShowError = !success && setupError && !readOnly && kind === GitHubAppKind.COMMIT_SIGNING
     return (
         <Container>
             <H3>
@@ -85,7 +89,7 @@ export const CommitSigningIntegrations: React.FunctionComponent<
                         GitHub App {appName?.length ? `"${appName}" ` : ''}successfully connected.
                     </DismissibleAlert>
                 )}
-                {!success && setupError && !readOnly && <GitHubAppFailureAlert error={setupError} />}
+                {shouldShowError && <GitHubAppFailureAlert error={setupError} />}
                 <ConnectionList as="ul" className="list-group" aria-label="commit signing integrations">
                     {connection?.nodes?.map(node =>
                         node.supportsCommitSigning ? (