Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bing): add the image generation function #381

Merged
merged 11 commits into from
May 31, 2023
Merged

feat(bing): add the image generation function #381

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c83cb7e
import '@timefox/bic-sydney' module to enbale BingAIClient's image ge…
May 23, 2023
afc23df
adding options for setting 'x-forwarded-for' and whether to enable im…
May 23, 2023
f740a62
make a slight optimization for bingAiClient's image generation function.
May 24, 2023
a4c1e5f
fix eslint error Trailing spaces not allowed no-trailing-spaces.
May 24, 2023
80d46f9
Fixed several issues pointed out by waylaidwanderer .
May 28, 2023
7eba01a
Merge branch 'main' into main
timefox May 28, 2023
e2d9888
Remove duplicate key 'cookie'.
May 28, 2023
5a0c55a
Fix eslint error Trailing spaces not allowed no-trailing-spaces.
May 28, 2023
e528787
Use the try-catch-await pattern for waiting for the image generation …
May 29, 2023
ef84a87
Do a mirror optimization.
May 29, 2023
7837b58
Fixed the crash when bic throw an error.
May 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions bin/cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ if (settings.storageFilePath && !settings.cacheOptions.store) {
settings.cacheOptions.store = new KeyvFile({ filename: settings.storageFilePath });
}

// Disable the image generation in cli mode always.
settings.bingAiClient.features = settings.bingAiClient.features || {};
settings.bingAiClient.features.genImage = false;

let conversationData = {};

const availableCommands = [
Expand Down
19 changes: 19 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
"dependencies": {
"@dqbd/tiktoken": "^1.0.2",
"@fastify/cors": "^8.2.0",
"@timefox/bic-sydney": "^1.1.2",
"@waylaidwanderer/fastify-sse-v2": "^3.1.0",
"@waylaidwanderer/fetch-event-source": "^3.0.1",
"boxen": "^7.0.1",
Expand Down
7 changes: 7 additions & 0 deletions settings.example.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@ export default {
cookies: '',
// A proxy string like "http://<ip>:<port>"
proxy: '',
// (Optional) Set 'x-forwarded-for' for the request. You can use a fixed IPv4 address or specify a range using CIDR notation,
// and the program will randomly select an address within that range. The 'x-forwarded-for' is not used by default now.
// xForwardedFor: '13.104.0.0/14',
// (Optional) Set 'genImage' to true to enable bing to create images for you. It's disabled by default.
// features: {
// genImage: true,
// },
// (Optional) Set to true to enable `console.debug()` logging
debug: false,
},
Expand Down
96 changes: 86 additions & 10 deletions src/BingAIClient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import WebSocket from 'ws';
import Keyv from 'keyv';
import { ProxyAgent } from 'undici';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { BingImageCreator } from '@timefox/bic-sydney';

/**
* https://stackoverflow.com/a/58326357
Expand Down Expand Up @@ -39,9 +40,40 @@ export default class BingAIClient {
this.options = {
...options,
host: options.host || 'https://www.bing.com',
xForwardedFor: this.constructor.getValidIPv4(options.xForwardedFor),
features: {
genImage: options?.features?.genImage || false,
},
};
}
this.debug = this.options.debug;
if (this.options.features.genImage) {
this.bic = new BingImageCreator(this.options);
}
}

static getValidIPv4(ip) {
const match = !ip
|| ip.match(/^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$/);
if (match) {
if (match[5]) {
const mask = parseInt(match[5], 10);
let [a, b, c, d] = ip.split('.').map(x => parseInt(x, 10));
// eslint-disable-next-line no-bitwise
const max = (1 << (32 - mask)) - 1;
const rand = Math.floor(Math.random() * max);
d += rand;
c += Math.floor(d / 256);
d %= 256;
b += Math.floor(c / 256);
c %= 256;
a += Math.floor(b / 256);
b %= 256;
return `${a}.${b}.${c}.${d}`;
}
return ip;
}
return undefined;
}

async createNewConversation() {
Expand All @@ -50,11 +82,11 @@ export default class BingAIClient {
accept: 'application/json',
'accept-language': 'en-US,en;q=0.9',
'content-type': 'application/json',
'sec-ch-ua': '"Chromium";v="112", "Microsoft Edge";v="112", "Not:A-Brand";v="99"',
'sec-ch-ua': '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
'sec-ch-ua-arch': '"x86"',
'sec-ch-ua-bitness': '"64"',
'sec-ch-ua-full-version': '"112.0.1722.7"',
'sec-ch-ua-full-version-list': '"Chromium";v="112.0.5615.20", "Microsoft Edge";v="112.0.1722.7", "Not:A-Brand";v="99.0.0.0"',
'sec-ch-ua-full-version': '"113.0.1774.50"',
'sec-ch-ua-full-version-list': '"Microsoft Edge";v="113.0.1774.50", "Chromium";v="113.0.5672.127", "Not-A.Brand";v="24.0.0.0"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-model': '""',
'sec-ch-ua-platform': '"Windows"',
Expand All @@ -64,11 +96,13 @@ export default class BingAIClient {
'sec-fetch-site': 'same-origin',
'x-ms-client-request-id': crypto.randomUUID(),
'x-ms-useragent': 'azsdk-js-api-client-factory/1.0.0-beta.1 core-rest-pipeline/1.10.0 OS/Win32',
cookie: this.options.cookies || (this.options.userToken ? `_U=${this.options.userToken}` : undefined),
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.50',
cookie: this.options.cookies || `_U=${this.options.userToken}`,
waylaidwanderer marked this conversation as resolved.
Show resolved Hide resolved
Referer: 'https://www.bing.com/search?q=Bing+AI&showconv=1&FORM=hpcodx',
'Referrer-Policy': 'origin-when-cross-origin',
// Workaround for request being blocked due to geolocation
'x-forwarded-for': '1.1.1.1',
// 'x-forwarded-for': '1.1.1.1', // 1.1.1.1 seems to no longer work.
...(this.xForwardedFor ? { 'x-forwarded-for': this.xForwardedFor } : {}),
waylaidwanderer marked this conversation as resolved.
Show resolved Hide resolved
},
};
if (this.options.proxy) {
Expand Down Expand Up @@ -309,6 +343,7 @@ export default class BingAIClient {
'cricinfov2',
'dv3sugg',
'nojbfedge',
...((toneStyle === 'creative' && this.options.features.genImage) ? ['gencontentv3'] : []),
],
sliceIds: [
'222dtappid',
Expand Down Expand Up @@ -377,6 +412,7 @@ export default class BingAIClient {
reject(new Error('Request aborted'));
});

let bicIframe;
ws.on('message', (data) => {
const objects = data.toString().split('');
const events = objects.map((object) => {
Expand All @@ -399,6 +435,15 @@ export default class BingAIClient {
if (!messages?.length || messages[0].author !== 'bot') {
return;
}
if (messages[0]?.contentType === 'IMAGE') {
// You will never get a message of this type without 'gencontentv3' being on.
bicIframe = this.bic.genImageIframeSsr(
messages[0].text,
messages[0].messageId,
progress => (progress?.contentIframe ? onProgress(progress?.contentIframe) : null),
);
return;
}
const updatedText = messages[0].text;
if (!updatedText || updatedText === replySoFar) {
return;
Expand All @@ -423,7 +468,7 @@ export default class BingAIClient {
return;
}
const messages = event.item?.messages || [];
const eventMessage = messages.length ? messages[messages.length - 1] : null;
let eventMessage = messages.length ? messages[messages.length - 1] : null;
if (event.item?.result?.error) {
if (this.debug) {
console.debug(event.item.result.value, event.item.result.message);
Expand Down Expand Up @@ -468,10 +513,36 @@ export default class BingAIClient {
// delete useless suggestions from moderation filter
delete eventMessage.suggestedResponses;
}
resolve({
message: eventMessage,
conversationExpiryTime: event?.item?.conversationExpiryTime,
});
if (bicIframe) {
// the last messages will be a image creation event if bicIframe is present.
let i = messages.length - 1;
while (eventMessage?.contentType === 'IMAGE' && i > 0) {
eventMessage = messages[i -= 1];
}

// wait for bicIframe to be completed.
bicIframe.then(async (result) => {
// The frame can be large, only put it into adaptiveCards.
eventMessage.adaptiveCards[0].body[0].text += result;
resolve({
message: eventMessage,
conversationExpiryTime: event?.item?.conversationExpiryTime,
});
}).catch((error) => {
eventMessage.text += `<br>${error}`;
eventMessage.adaptiveCards[0].body[0].text = eventMessage.text;
resolve({
message: eventMessage,
conversationExpiryTime: event?.item?.conversationExpiryTime,
});
waylaidwanderer marked this conversation as resolved.
Show resolved Hide resolved
});
} else {
// if there is no bicIframe, we resolve it normally.
resolve({
message: eventMessage,
conversationExpiryTime: event?.item?.conversationExpiryTime,
});
}
// eslint-disable-next-line no-useless-return
return;
}
Expand All @@ -484,6 +555,11 @@ export default class BingAIClient {
return;
}
default:
if (event?.error) {
clearTimeout(messageTimeout);
this.constructor.cleanupWebSocketConnection(ws);
reject(new Error(`Event Type('${event.type}'): ${event.error}`));
}
// eslint-disable-next-line no-useless-return
return;
}
Expand Down