fix(core): use backend prompts (#6542)

This commit is contained in:
pengx17
2024-04-12 11:15:38 +00:00
committed by Peng Xiao
parent 2336638996
commit 9b620ecbc9
9 changed files with 482 additions and 284 deletions

View File

@@ -73,6 +73,18 @@ export const prompts: Prompt[] = [
}, },
], ],
}, },
{
name: 'Explain this',
action: 'Explain this',
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'assistant',
content:
'Explain the following content in a clear and concise manner, ensuring that the information is easy to understand and provides a comprehensive overview of the topic:\n\n{{content}}',
},
],
},
{ {
name: 'Explain this image', name: 'Explain this image',
action: 'Explain this image', action: 'Explain this image',
@@ -167,6 +179,18 @@ export const prompts: Prompt[] = [
}, },
], ],
}, },
{
name: 'Write outline',
action: 'Write outline',
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'assistant',
content:
'Write an outline based on the following content, organizing the main points, subtopics, and structure:\n\n{{content}}',
},
],
},
{ {
name: 'Change tone to', name: 'Change tone to',
action: 'Change tone', action: 'Change tone',
@@ -192,6 +216,18 @@ export const prompts: Prompt[] = [
}, },
], ],
}, },
{
name: 'Brainstorm mindmap',
action: 'Brainstorm mindmap',
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'assistant',
content:
'Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: \n\n{{content}}',
},
],
},
{ {
name: 'Improve writing for it', name: 'Improve writing for it',
action: 'Improve writing for it', action: 'Improve writing for it',
@@ -329,4 +365,26 @@ export const prompts: Prompt[] = [
}, },
], ],
}, },
{
name: 'Make it longer',
action: 'Make it longer',
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'assistant',
content: 'Make the following content longer:\n\n{{content}}',
},
],
},
{
name: 'Make it shorter',
action: 'Make it shorter',
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'assistant',
content: 'Make the following content shorter:\n\n{{content}}',
},
],
},
]; ];

View File

@@ -0,0 +1,140 @@
import {
createCopilotMessageMutation,
createCopilotSessionMutation,
fetcher,
getBaseUrl,
getCopilotHistoriesQuery,
getCopilotSessionsQuery,
type GraphQLQuery,
type RequestOptions,
} from '@affine/graphql';
type OptionsField<T extends GraphQLQuery> =
RequestOptions<T>['variables'] extends { options: infer U } ? U : never;
export class CopilotClient {
readonly backendUrl = getBaseUrl();
async createSession(
options: OptionsField<typeof createCopilotSessionMutation>
) {
const res = await fetcher({
query: createCopilotSessionMutation,
variables: {
options,
},
});
return res.createCopilotSession;
}
async createMessage(
options: OptionsField<typeof createCopilotMessageMutation>
) {
const res = await fetcher({
query: createCopilotMessageMutation,
variables: {
options,
},
});
return res.createCopilotMessage;
}
async getSessions(workspaceId: string) {
const res = await fetcher({
query: getCopilotSessionsQuery,
variables: {
workspaceId,
},
});
return res.currentUser?.copilot;
}
async getHistories(
workspaceId: string,
docId?: string,
options?: OptionsField<typeof getCopilotHistoriesQuery>
) {
const res = await fetcher({
query: getCopilotHistoriesQuery,
variables: {
workspaceId,
docId,
options,
},
});
return res.currentUser?.copilot?.histories;
}
async textToText(message: string, sessionId: string) {
const res = await fetch(
`${this.backendUrl}/api/copilot/chat/${sessionId}?message=${encodeURIComponent(message)}`
);
if (!res.ok) return;
return res.text();
}
textToTextStream(message: string, sessionId: string) {
return new EventSource(
`${this.backendUrl}/api/copilot/chat/${sessionId}/stream?message=${encodeURIComponent(message)}`
);
}
chatText({
sessionId,
messageId,
message,
}: {
sessionId: string;
messageId?: string;
message?: string;
}) {
if (messageId && message) {
throw new Error('Only one of messageId or message can be provided');
} else if (!messageId && !message) {
throw new Error('Either messageId or message must be provided');
}
const url = new URL(`${this.backendUrl}/api/copilot/chat/${sessionId}`);
if (messageId) {
url.searchParams.set('messageId', messageId);
}
if (message) {
url.searchParams.set('message', message);
}
return fetch(url.toString());
}
// Text or image to text
chatTextStream({
sessionId,
messageId,
message,
}: {
sessionId: string;
messageId?: string;
message?: string;
}) {
if (messageId && message) {
throw new Error('Only one of messageId or message can be provided');
} else if (!messageId && !message) {
throw new Error('Either messageId or message must be provided');
}
const url = new URL(
`${this.backendUrl}/api/copilot/chat/${sessionId}/stream`
);
if (messageId) {
url.searchParams.set('messageId', messageId);
}
if (message) {
url.searchParams.set('message', message);
}
return new EventSource(url.toString());
}
// Text or image to images
imagesStream(messageId: string, sessionId: string) {
return new EventSource(
`${this.backendUrl}/api/copilot/chat/${sessionId}/images?messageId=${messageId}`
);
}
}

View File

@@ -0,0 +1,35 @@
// manually synced with packages/backend/server/src/data/migrations/utils/prompts.ts
// todo: automate this
export const promptKeys = [
'debug:chat:gpt4',
'debug:action:gpt4',
'debug:action:vision4',
'debug:action:dalle3',
'debug:action:fal-sd15',
'Summary',
'Summary the webpage',
'Explain this',
'Explain this image',
'Explain this code',
'Translate to',
'Write an article about this',
'Write a twitter about this',
'Write a poem about this',
'Write a blog post about this',
'Write outline',
'Change tone to',
'Brainstorm ideas about this',
'Brainstorm mindmap',
'Improve writing for it',
'Improve grammar for it',
'Fix spelling for it',
'Find action items from it',
'Check code error',
'Create a presentation',
'Create headings',
'Make it real',
'Make it longer',
'Make it shorter',
] as const;
export type PromptKey = (typeof promptKeys)[number];

View File

@@ -1,284 +1,193 @@
import { assertExists } from '@blocksuite/global/utils';
import { AIProvider } from '@blocksuite/presets'; import { AIProvider } from '@blocksuite/presets';
import { imageToTextStream, textToTextStream } from './request'; import { textToText } from './request';
export function setupAIProvider() { export function setupAIProvider() {
AIProvider.provideAction('chat', options => { AIProvider.provideAction('chat', options => {
assertExists(options.stream); return textToText({
return textToTextStream({ ...options,
docId: options.docId, content: options.input,
workspaceId: options.workspaceId, promptName: 'debug:chat:gpt4',
prompt: options.input,
}); });
}); });
AIProvider.provideAction('summary', options => { AIProvider.provideAction('summary', options => {
assertExists(options.stream); return textToText({
const prompt = ` ...options,
Summarize the key points from the following content in a clear and concise manner, content: options.input,
suitable for a reader who is seeking a quick understanding of the original content. promptName: 'Summary',
Ensure to capture the main ideas and any significant details without unnecessary elaboration:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('translate', options => { AIProvider.provideAction('translate', options => {
assertExists(options.stream); return textToText({
const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content: ...options,
promptName: 'Translate to',
${options.input} content: options.input,
`; params: {
return textToTextStream({ language: options.lang,
docId: options.docId, },
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('changeTone', options => { AIProvider.provideAction('changeTone', options => {
assertExists(options.stream); return textToText({
const prompt = `Change the tone of the following content to ${options.tone}: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Change tone to',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('improveWriting', options => { AIProvider.provideAction('improveWriting', options => {
assertExists(options.stream); return textToText({
const prompt = `Improve the writing of the following content: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Improve writing for it',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('improveGrammar', options => { AIProvider.provideAction('improveGrammar', options => {
assertExists(options.stream); return textToText({
const prompt = `Improve the grammar of the following content: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Improve grammar for it',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('fixSpelling', options => { AIProvider.provideAction('fixSpelling', options => {
assertExists(options.stream); return textToText({
const prompt = `Fix the spelling of the following content: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Fix spelling for it',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('createHeadings', options => { AIProvider.provideAction('createHeadings', options => {
assertExists(options.stream); return textToText({
const prompt = `Create headings for the following content: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Create headings',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('makeLonger', options => { AIProvider.provideAction('makeLonger', options => {
assertExists(options.stream); return textToText({
const prompt = `Make the following content longer: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Make it longer',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('makeShorter', options => { AIProvider.provideAction('makeShorter', options => {
assertExists(options.stream); return textToText({
const prompt = `Make the following content shorter: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Make it shorter',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('checkCodeErrors', options => { AIProvider.provideAction('checkCodeErrors', options => {
assertExists(options.stream); return textToText({
const prompt = `Check the code errors in the following content and provide the corrected version: ...options,
content: options.input,
${options.input} promptName: 'Check code error',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('explainCode', options => { AIProvider.provideAction('explainCode', options => {
assertExists(options.stream); return textToText({
const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes: ...options,
content: options.input,
${options.input} promptName: 'Explain this code',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('writeArticle', options => { AIProvider.provideAction('writeArticle', options => {
assertExists(options.stream); return textToText({
const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow: ...options,
content: options.input,
${options.input} promptName: 'Write an article about this',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('writeTwitterPost', options => { AIProvider.provideAction('writeTwitterPost', options => {
assertExists(options.stream); return textToText({
const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging: ...options,
content: options.input,
${options.input} promptName: 'Write a twitter about this',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('writePoem', options => { AIProvider.provideAction('writePoem', options => {
assertExists(options.stream); return textToText({
const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm: ...options,
content: options.input,
${options.input} promptName: 'Write a poem about this',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('writeOutline', options => { AIProvider.provideAction('writeOutline', options => {
assertExists(options.stream); return textToText({
const prompt = `Write an outline from the following content in Markdown: ${options.input}`; ...options,
content: options.input,
return textToTextStream({ promptName: 'Write outline',
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('writeBlogPost', options => { AIProvider.provideAction('writeBlogPost', options => {
assertExists(options.stream); return textToText({
const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective: ...options,
content: options.input,
${options.input} promptName: 'Write a blog post about this',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('brainstorm', options => { AIProvider.provideAction('brainstorm', options => {
assertExists(options.stream); return textToText({
const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches: ...options,
content: options.input,
${options.input} promptName: 'Brainstorm ideas about this',
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('findActions', options => { AIProvider.provideAction('findActions', options => {
assertExists(options.stream); return textToText({
const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`; ...options,
content: options.input,
return textToTextStream({ promptName: 'Find action items from it',
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
});
});
AIProvider.provideAction('writeOutline', options => {
assertExists(options.stream);
const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('brainstormMindmap', options => { AIProvider.provideAction('brainstormMindmap', options => {
assertExists(options.stream); return textToText({
const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Brainstorm mindmap',
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('explain', options => { AIProvider.provideAction('explain', options => {
assertExists(options.stream); return textToText({
const prompt = `Explain the following content in Markdown: ${options.input}`; ...options,
content: options.input,
return textToTextStream({ promptName: 'Explain this',
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
}); });
}); });
AIProvider.provideAction('explainImage', options => { AIProvider.provideAction('explainImage', options => {
assertExists(options.stream); return textToText({
const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`; ...options,
return textToTextStream({ content: options.input,
docId: options.docId, promptName: 'Explain this image',
workspaceId: options.workspaceId,
prompt,
attachments: options.attachments,
}); });
}); });
AIProvider.provideAction('makeItReal', options => { AIProvider.provideAction('makeItReal', options => {
assertExists(options.stream); return textToText({
const promptName = 'Make it real'; ...options,
return imageToTextStream({ promptName: 'Make it real',
promptName, // @ts-expect-error todo: fix this after blocksuite bump
docId: options.docId,
workspaceId: options.workspaceId,
params: options.params, params: options.params,
attachments: options.attachments,
content: content:
options.content || options.content ||
'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?', 'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?',

View File

@@ -1,107 +1,135 @@
import { getBaseUrl } from '@affine/graphql'; import { toTextStream } from '@blocksuite/presets';
import { CopilotClient, toTextStream } from '@blocksuite/presets';
const TIMEOUT = 500000; import { CopilotClient } from './copilot-client';
import type { PromptKey } from './prompt';
export function textToTextStream({ const TIMEOUT = 50000;
docId,
workspaceId, const client = new CopilotClient();
prompt,
attachments, function readBlobAsURL(blob: Blob) {
params, return new Promise<string>((resolve, reject) => {
}: { const reader = new FileReader();
reader.onload = e => {
if (typeof e.target?.result === 'string') {
resolve(e.target.result);
} else {
reject();
}
};
reader.onerror = reject;
reader.readAsDataURL(blob);
});
}
export type TextToTextOptions = {
docId: string; docId: string;
workspaceId: string; workspaceId: string;
prompt: string; promptName: PromptKey;
attachments?: string[]; content?: string;
params?: string; attachments?: (string | Blob)[];
}): BlockSuitePresets.TextStream { params?: Record<string, string>;
const client = new CopilotClient(getBaseUrl()); timeout?: number;
return { stream?: boolean;
[Symbol.asyncIterator]: async function* () { };
const hasAttachments = attachments && attachments.length > 0;
const session = await client.createSession({ async function createSessionMessage({
workspaceId, docId,
docId, workspaceId,
promptName: hasAttachments ? 'debug:action:vision4' : 'Summary', promptName,
}); content,
if (hasAttachments) { attachments,
const messageId = await client.createMessage({ params,
sessionId: session, }: TextToTextOptions) {
content: prompt, const hasAttachments = attachments && attachments.length > 0;
const session = await client.createSession({
workspaceId,
docId,
promptName,
});
if (hasAttachments) {
const normalizedAttachments = await Promise.all(
attachments.map(async attachment => {
if (typeof attachment === 'string') {
return attachment;
}
const url = await readBlobAsURL(attachment);
return url;
})
);
const messageId = await client.createMessage({
sessionId: session,
content,
attachments: normalizedAttachments,
params,
});
return {
messageId,
session,
};
} else if (content) {
return {
message: content,
session,
};
} else {
throw new Error('No content or attachments provided');
}
}
export function textToText({
docId,
workspaceId,
promptName,
content,
attachments,
params,
stream,
timeout = TIMEOUT,
}: TextToTextOptions) {
if (stream) {
return {
[Symbol.asyncIterator]: async function* () {
const message = await createSessionMessage({
docId,
workspaceId,
promptName,
content,
attachments, attachments,
params, params,
}); });
const eventSource = client.textStream(messageId, session);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
} else {
const eventSource = client.textToTextStream(prompt, session);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
}
},
};
}
// Image to text(html) const eventSource = client.chatTextStream({
export function imageToTextStream({ sessionId: message.session,
docId, messageId: message.messageId,
workspaceId, message: message.message,
promptName, });
...options yield* toTextStream(eventSource, { timeout: timeout });
}: { },
docId: string; };
workspaceId: string; } else {
promptName: string; return Promise.race([
params?: string; timeout
content: string; ? new Promise((_res, rej) => {
attachments?: string[]; setTimeout(() => {
}) { rej(new Error('Timeout'));
const client = new CopilotClient(getBaseUrl()); }, timeout);
return { })
[Symbol.asyncIterator]: async function* () { : null,
const sessionId = await client.createSession({ createSessionMessage({
workspaceId,
docId, docId,
promptName,
});
const messageId = await client.createMessage({
sessionId,
...options,
});
const eventSource = client.textStream(messageId, sessionId);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
},
};
}
// Image to images
export function imageToImagesStream({
docId,
workspaceId,
promptName,
...options
}: {
docId: string;
workspaceId: string;
promptName: string;
content: string;
params?: string;
attachments?: string[];
}) {
const client = new CopilotClient(getBaseUrl());
return {
[Symbol.asyncIterator]: async function* () {
const sessionId = await client.createSession({
workspaceId, workspaceId,
docId,
promptName, promptName,
}); content,
const messageId = await client.createMessage({ attachments,
sessionId, params,
...options, }).then(message => {
}); return client.chatText({
const eventSource = client.imagesStream(messageId, sessionId); sessionId: message.session,
yield* toTextStream(eventSource, { timeout: TIMEOUT }); messageId: message.messageId,
}, message: message.message,
}; });
}),
]);
}
} }

View File

@@ -16,7 +16,7 @@ config:
Decimal: number Decimal: number
UUID: string UUID: string
ID: string ID: string
JSON: string JSON: Record<string, string>
Upload: File Upload: File
SafeInt: number SafeInt: number
overwrite: true overwrite: true

View File

@@ -0,0 +1,3 @@
mutation createCopilotMessage($options: CreateChatMessageInput!) {
createCopilotMessage(options: $options)
}

View File

@@ -144,6 +144,17 @@ mutation createCheckoutSession($input: CreateCheckoutSessionInput!) {
}`, }`,
}; };
export const createCopilotMessageMutation = {
id: 'createCopilotMessageMutation' as const,
operationName: 'createCopilotMessage',
definitionName: 'createCopilotMessage',
containsFile: false,
query: `
mutation createCopilotMessage($options: CreateChatMessageInput!) {
createCopilotMessage(options: $options)
}`,
};
export const createCopilotSessionMutation = { export const createCopilotSessionMutation = {
id: 'createCopilotSessionMutation' as const, id: 'createCopilotSessionMutation' as const,
operationName: 'createCopilotSession', operationName: 'createCopilotSession',

View File

@@ -29,7 +29,7 @@ export interface Scalars {
/** A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date-time format. */ /** A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date-time format. */
DateTime: { input: string; output: string }; DateTime: { input: string; output: string };
/** The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). */ /** The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). */
JSON: { input: string; output: string }; JSON: { input: Record<string, string>; output: Record<string, string> };
/** The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. */ /** The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. */
SafeInt: { input: number; output: number }; SafeInt: { input: number; output: number };
/** The `Upload` scalar type represents a file upload. */ /** The `Upload` scalar type represents a file upload. */
@@ -240,6 +240,15 @@ export type CreateCheckoutSessionMutation = {
createCheckoutSession: string; createCheckoutSession: string;
}; };
export type CreateCopilotMessageMutationVariables = Exact<{
options: CreateChatMessageInput;
}>;
export type CreateCopilotMessageMutation = {
__typename?: 'Mutation';
createCopilotMessage: string;
};
export type CreateCopilotSessionMutationVariables = Exact<{ export type CreateCopilotSessionMutationVariables = Exact<{
options: CreateChatSessionInput; options: CreateChatSessionInput;
}>; }>;
@@ -1214,6 +1223,11 @@ export type Mutations =
variables: CreateCheckoutSessionMutationVariables; variables: CreateCheckoutSessionMutationVariables;
response: CreateCheckoutSessionMutation; response: CreateCheckoutSessionMutation;
} }
| {
name: 'createCopilotMessageMutation';
variables: CreateCopilotMessageMutationVariables;
response: CreateCopilotMessageMutation;
}
| { | {
name: 'createCopilotSessionMutation'; name: 'createCopilotSessionMutation';
variables: CreateCopilotSessionMutationVariables; variables: CreateCopilotSessionMutationVariables;