implemented images support for gpt4

README.md

@@ -93,6 +93,7 @@ Go to <b>"Manage my extensions"</b> on your browser, then click on <b>"Load unpa
 - <b>Infinite try</b>: click as much as you want on the question (don't forget to reset the question).
 - <b>Save history</b>: allows you to create a conversation with ChatGPT by saving the previous question with its answer. However, note that it can consume a significant number of tokens.
 - <b>Include images</b> (only work with gpt-4): allows you to include the images from the question to be send to the chatgpt api. However, note that it can consume a significant number of tokens.
+  <br/> ![Images](./assets/images.gif)
 
 ## Internal Features
 

TODO.md

@@ -1,8 +1,5 @@
 # TODO
 
 - Historic for questions (implemented but need testing)
-- Image integration for gpt-4 (implemented but need testing)
-
 - Better prompt (Fixe put in order question, Fixe calculation question)
-- Support math equation from image stocked in the `data-mathml` attribute
 - Better webstore assets

extension/popup/js/gptVersion.js

@@ -3,15 +3,20 @@
 const modelInput = document.querySelector('#model');
 const imagesIntegrationLine = document.querySelector('#includeImages-line');
 
-// Check if the gpt version is at least 4 to show the option 'Include images'
-modelInput.addEventListener('input', function () {
+/**
+ * Check if the gpt version is at least 4 to show the option 'Include images'
+ */
+function checkCanIncludeImages() {
   const version = modelInput.value;
   if (isCurrentVersionSupportingImages(version)) {
     imagesIntegrationLine.style.display = 'flex';
   } else {
     imagesIntegrationLine.style.display = 'none';
   }
-});
+}
+
+checkCanIncludeImages();
+modelInput.addEventListener('input', checkCanIncludeImages);
 
 /**
  * Get the last ChatGPT version

src/core/get-response.ts

@@ -2,14 +2,20 @@ import type Config from '@typing/config';
 import type GPTAnswer from '@typing/gptAnswer';
 import imageToBase64 from '@utils/image-to-base64';
 import normalizeText from '@utils/normalize-text';
-import isCurrentVersionSupportingImages from '@utils/version-support-images';
+import isGPTModelGreaterOrEqualTo4 from '@utils/version-support-images';
 
 type Content =
   | string
-  | Array<{
-      type: CONTENT_TYPE;
-      content: string;
-    }>;
+  | Array<
+      | {
+          type: CONTENT_TYPE.TEXT;
+          text: string;
+        }
+      | {
+          type: CONTENT_TYPE.IMAGE;
+          image_url: { url: string };
+        }
+    >;
 
 type History = {
   url: string | null;
@@ -63,8 +69,8 @@ async function getContent(
   const imagesElements = questionElement.querySelectorAll('img');
 
   if (
-    !config.includeImages ||
-    !isCurrentVersionSupportingImages(config.model) ||
+    config.includeImages &&
+    isGPTModelGreaterOrEqualTo4(config.model) &&
     imagesElements.length === 0
   ) {
     return question;
@@ -79,14 +85,14 @@ async function getContent(
   for (const result of filteredResults) {
     content.push({
       type: CONTENT_TYPE.IMAGE,
-      content: result
+      image_url: { url: result }
     });
   }
 
   if (content.length > 0) {
     content.push({
       type: CONTENT_TYPE.TEXT,
-      content: question
+      text: question
     });
   } else {
     content = question;
@@ -133,7 +139,7 @@ async function getChatGPTResponse(
       temperature: 0.8,
       top_p: 1.0,
       presence_penalty: 1.0,
-      stop: null
+      ...(isGPTModelGreaterOrEqualTo4(config.model) ? { max_tokens: 1000 } : { stop: null }) // look like that on 3.5 we can say stop do null but not on gpt >= 4
     })
   });
 

src/utils/version-support-images.ts

@@ -1,9 +1,9 @@
 /**
- * Check if the current ChatGPT version support image sending
+ * Check if the current ChatGPT version is greater or equal to 4
  * @param version
  * @returns
  */
-function isCurrentVersionSupportingImages(version: string): boolean {
+function isGPTModelGreaterOrEqualTo4(version: string): boolean {
   const versionNumber = version.match(/gpt-(\d+)/);
   if (!versionNumber?.[1]) {
     return false;
@@ -11,4 +11,4 @@ function isCurrentVersionSupportingImages(version: string): boolean {
   return Number(versionNumber[1]) >= 4;
 }
 
-export default isCurrentVersionSupportingImages;
+export default isGPTModelGreaterOrEqualTo4;

test/fake-moodle/css/style.css

@@ -65,3 +65,7 @@ textarea {
   white-space: pre-wrap;
   overflow-y: auto;
 }
+
+img {
+  height: 5rem;
+}

test/fake-moodle/index.html

@@ -269,7 +269,7 @@
       </div>
     </section>
 
-    <!-- Clipboard -->
+    <!-- Contenteditable -->
     <section class="formulation">
       <div class="qtext">
         <p>
@@ -281,5 +281,20 @@
         <div contenteditable="true" class="editable"></div>
       </div>
     </section>
+
+    <!-- Images -->
+    <section class="formulation">
+      <p class="accesshide" style="color: red">
+        Warning ! Only work with gpt-4 and if "includes images" is activate
+      </p>
+      <div class="qtext">
+        <p>Tell me what you see on thoses images</p>
+        <img src="./assets/cat1.jpg" />
+        <img src="./assets/cat2.jpg" />
+      </div>
+      <div>
+        <div contenteditable="true" class="editable"></div>
+      </div>
+    </section>
   </body>
 </html>